From e047bf68e0508176f2c9ba3ab15a7fa1ced6c1be Mon Sep 17 00:00:00 2001 From: gojomo Date: Mon, 11 May 2009 22:56:36 +0000 Subject: [PATCH] copy 'springy' branch to heritrix3 --- .classpath | 63 + .project | 17 + commons/.classpath | 30 + commons/.project | 14 + commons/.settings/org.eclipse.jdt.core.prefs | 5 + commons/pom.xml | 321 ++ .../org/apache/commons/httpclient/Cookie.java | 575 ++++ .../commons/httpclient/HttpConnection.java | 1406 ++++++++ .../commons/httpclient/HttpMethodBase.java | 2422 +++++++++++++ .../apache/commons/httpclient/HttpParser.java | 236 ++ .../apache/commons/httpclient/HttpState.java | 717 ++++ .../commons/httpclient/cookie/CookieSpec.java | 264 ++ .../httpclient/cookie/CookieSpecBase.java | 716 ++++ .../httpclient/cookie/IgnoreCookiesSpec.java | 160 + .../pool/impl/FairGenericObjectPool.java | 310 ++ .../pool/impl/FairGenericObjectPoolTest.java | 122 + .../commons/pool/impl/GenericObjectPool.java | 1312 +++++++ .../main/java/org/archive/bdb/BdbModule.java | 637 ++++ .../checkpointing/CheckpointInputStream.java | 77 + .../checkpointing/CheckpointRecovery.java | 47 + .../archive/checkpointing/Checkpointable.java | 44 + .../archive/checkpointing/Checkpointer.java | 120 + .../DefaultCheckpointRecovery.java | 109 + .../archive/checkpointing/RecoverAction.java | 45 + .../ConfigurableX509TrustManager.java | 192 ++ .../httpclient/HttpRecorderGetMethod.java | 124 + .../httpclient/HttpRecorderMethod.java | 111 + .../httpclient/HttpRecorderPostMethod.java | 86 + .../SingleHttpConnectionManager.java | 76 + .../ThreadLocalHttpConnectionManager.java | 291 ++ .../java/org/archive/httpclient/package.html | 24 + .../main/java/org/archive/io/Arc2Warc.java | 208 ++ .../org/archive/io/ArchiveFileConstants.java | 118 + .../java/org/archive/io/ArchiveReader.java | 779 +++++ .../org/archive/io/ArchiveReaderFactory.java | 326 ++ .../java/org/archive/io/ArchiveRecord.java | 413 +++ .../org/archive/io/ArchiveRecordHeader.java | 109 + .../org/archive/io/ArraySeekInputStream.java | 110 + .../archive/io/BufferedSeekInputStream.java | 221 ++ .../archive/io/ByteReplayCharSequence.java | 363 ++ .../java/org/archive/io/CharSubSequence.java | 94 + .../archive/io/CompositeFileInputStream.java | 103 + .../org/archive/io/CompositeFileReader.java | 46 + .../java/org/archive/io/CrawlerJournal.java | 271 ++ .../archive/io/DefaultWriterPoolSettings.java | 150 + .../src/main/java/org/archive/io/Endian.java | 129 + .../org/archive/io/GenerationFileHandler.java | 136 + .../main/java/org/archive/io/GzipHeader.java | 344 ++ .../org/archive/io/GzippedInputStream.java | 422 +++ .../org/archive/io/HeaderedArchiveRecord.java | 422 +++ .../archive/io/LoudObjectOutputStream.java | 70 + .../io/MultiByteReplayCharSequence.java | 346 ++ .../org/archive/io/NoGzipMagicException.java | 36 + .../io/ObjectPlusFilesInputStream.java | 143 + .../io/ObjectPlusFilesOutputStream.java | 140 + .../org/archive/io/OriginSeekInputStream.java | 125 + .../archive/io/RandomAccessInputStream.java | 184 + .../archive/io/RandomAccessOutputStream.java | 74 + .../main/java/org/archive/io/ReadSource.java | 26 + .../org/archive/io/RecorderIOException.java | 43 + .../io/RecorderLengthExceededException.java | 44 + .../archive/io/RecorderTimeoutException.java | 42 + .../io/RecorderTooMuchHeaderException.java | 45 + .../org/archive/io/RecordingInputStream.java | 369 ++ .../org/archive/io/RecordingOutputStream.java | 625 ++++ .../archive/io/RecoverableIOException.java | 87 + .../io/RecyclingFastBufferedOutputStream.java | 150 + .../org/archive/io/ReplayCharSequence.java | 46 + .../org/archive/io/ReplayInputStream.java | 282 ++ .../archive/io/RepositionableInputStream.java | 139 + .../org/archive/io/SafeSeekInputStream.java | 128 + .../java/org/archive/io/SeekInputStream.java | 85 + .../main/java/org/archive/io/SeekReader.java | 88 + .../archive/io/SeekReaderCharSequence.java | 56 + .../org/archive/io/SinkHandlerLogThread.java | 38 + .../main/java/org/archive/io/UTF8Bytes.java | 41 + .../main/java/org/archive/io/Warc2Arc.java | 226 ++ .../main/java/org/archive/io/WriterPool.java | 267 ++ .../java/org/archive/io/WriterPoolMember.java | 538 +++ .../org/archive/io/WriterPoolSettings.java | 37 + .../java/org/archive/io/arc/ARC2WCDX.java | 243 ++ .../java/org/archive/io/arc/ARCConstants.java | 227 ++ .../java/org/archive/io/arc/ARCLocation.java | 43 + .../java/org/archive/io/arc/ARCReader.java | 551 +++ .../org/archive/io/arc/ARCReaderFactory.java | 427 +++ .../java/org/archive/io/arc/ARCRecord.java | 718 ++++ .../org/archive/io/arc/ARCRecordMetaData.java | 259 ++ .../java/org/archive/io/arc/ARCUtils.java | 244 ++ .../java/org/archive/io/arc/ARCWriter.java | 485 +++ .../org/archive/io/arc/ARCWriterPool.java | 79 + .../main/java/org/archive/io/arc/package.html | 9 + .../org/archive/io/warc/WARCConstants.java | 233 ++ .../java/org/archive/io/warc/WARCReader.java | 288 ++ .../archive/io/warc/WARCReaderFactory.java | 296 ++ .../java/org/archive/io/warc/WARCRecord.java | 225 ++ .../java/org/archive/io/warc/WARCWriter.java | 443 +++ .../org/archive/io/warc/WARCWriterPool.java | 76 + .../org/archive/io/warc/WARCWriterTest.java | 463 +++ .../java/org/archive/io/warc/package.html | 38 + .../main/java/org/archive/net/ClientFTP.java | 136 + .../archive/net/DownloadURLConnection.java | 134 + .../java/org/archive/net/FTPException.java | 62 + .../src/main/java/org/archive/net/LaxURI.java | 451 +++ .../java/org/archive/net/LaxURLCodec.java | 164 + .../java/org/archive/net/PublicSuffixes.java | 302 ++ .../src/main/java/org/archive/net/UURI.java | 447 +++ .../java/org/archive/net/UURIFactory.java | 846 +++++ .../java/org/archive/net/md5/Handler.java | 91 + .../org/archive/net/md5/Md5URLConnection.java | 38 + .../java/org/archive/net/rsync/Handler.java | 77 + .../archive/net/rsync/RsyncURLConnection.java | 57 + .../main/java/org/archive/net/s3/Handler.java | 40 + .../org/archive/net/s3/S3URLConnection.java | 88 + .../java/org/archive/queue/StoredQueue.java | 182 + .../org/archive/queue/StoredQueueTest.java | 152 + .../spring/BeanFieldsPatternValidator.java | 78 + .../java/org/archive/spring/ConfigFile.java | 67 + .../java/org/archive/spring/ConfigPath.java | 113 + .../archive/spring/ConfigPathConfigurer.java | 133 + .../org/archive/spring/ConfigPathEditor.java | 72 + .../java/org/archive/spring/ConfigString.java | 19 + .../archive/spring/HasKeyedProperties.java | 11 + .../java/org/archive/spring/HasValidator.java | 7 + .../org/archive/spring/KeyedProperties.java | 144 + .../org/archive/spring/OverlayContext.java | 38 + .../org/archive/spring/OverlayMapsSource.java | 31 + .../archive/spring/PathSharingContext.java | 220 ++ .../main/java/org/archive/spring/Sheet.java | 132 + .../java/org/archive/spring/WriteTarget.java | 8 + .../org/archive/state/ModuleTestBase.java | 209 ++ .../java/org/archive/surt/SURTTokenizer.java | 177 + .../main/java/org/archive/uid/Generator.java | 83 + .../org/archive/uid/GeneratorFactory.java | 79 + .../java/org/archive/uid/UUIDGenerator.java | 74 + .../main/java/org/archive/uid/package.html | 28 + .../org/archive/util/AbstractLongFPSet.java | 315 ++ .../java/org/archive/util/ArchiveUtils.java | 846 +++++ .../main/java/org/archive/util/Base32.java | 165 + .../org/archive/util/BenchmarkBlooms.java | 105 + .../java/org/archive/util/BloomFilter.java | 70 + .../org/archive/util/BloomFilter32bit.java | 223 ++ .../archive/util/BloomFilter32bitSplit.java | 251 ++ .../org/archive/util/BloomFilter32bp2.java | 235 ++ .../archive/util/BloomFilter32bp2Split.java | 262 ++ .../org/archive/util/BloomFilter64bit.java | 222 ++ .../java/org/archive/util/CachedBdbMap.java | 667 ++++ .../org/archive/util/CollectionUtils.java | 48 + .../java/org/archive/util/DNSJavaUtil.java | 80 + .../main/java/org/archive/util/DevUtils.java | 120 + .../main/java/org/archive/util/FileUtils.java | 612 ++++ .../java/org/archive/util/Histotable.java | 175 + .../org/archive/util/InetAddressUtil.java | 116 + .../util/InterruptibleCharSequence.java | 60 + .../util/InterruptibleCharSequenceTest.java | 121 + .../main/java/org/archive/util/Inverter.java | 44 + .../main/java/org/archive/util/IoUtils.java | 331 ++ .../java/org/archive/util/JavaLiterals.java | 148 + .../main/java/org/archive/util/JndiUtils.java | 182 + .../src/main/java/org/archive/util/LRU.java | 69 + .../java/org/archive/util/LongWrapper.java | 42 + .../java/org/archive/util/MimetypeUtils.java | 81 + .../java/org/archive/util/MultiReporter.java | 46 + .../org/archive/util/OneLineSimpleLogger.java | 137 + .../org/archive/util/PaddingStringBuffer.java | 170 + .../archive/util/PatternMatcherRecycler.java | 91 + .../java/org/archive/util/PreJ15Utils.java | 50 + .../java/org/archive/util/PrefixFinder.java | 150 + .../main/java/org/archive/util/PrefixSet.java | 74 + .../java/org/archive/util/ProcessUtils.java | 156 + .../util/ProgressStatisticsReporter.java | 42 + .../java/org/archive/util/PropertyUtils.java | 58 + .../main/java/org/archive/util/Recorder.java | 342 ++ .../java/org/archive/util/RecorderMarker.java | 38 + .../main/java/org/archive/util/Reporter.java | 47 + .../src/main/java/org/archive/util/SURT.java | 262 ++ .../main/java/org/archive/util/SubList.java | 150 + .../java/org/archive/util/SurtPrefixSet.java | 344 ++ .../main/java/org/archive/util/TestUtils.java | 131 + .../main/java/org/archive/util/TextUtils.java | 254 ++ .../org/archive/util/TimestampSerialno.java | 62 + .../java/org/archive/util/TmpDirTestCase.java | 122 + .../main/java/org/archive/util/Transform.java | 160 + .../java/org/archive/util/Transformer.java | 46 + .../org/archive/util/anvl/ANVLRecord.java | 333 ++ .../org/archive/util/anvl/ANVLRecords.java | 61 + .../java/org/archive/util/anvl/Element.java | 79 + .../java/org/archive/util/anvl/Label.java | 41 + .../org/archive/util/anvl/SubElement.java | 84 + .../java/org/archive/util/anvl/Value.java | 71 + .../java/org/archive/util/anvl/package.html | 42 + .../util/bdbje/EnhancedEnvironment.java | 87 + .../util/fingerprint/ArrayLongFPCache.java | 109 + .../archive/util/fingerprint/LongFPSet.java | 76 + .../util/fingerprint/LongFPSetCache.java | 77 + .../util/fingerprint/LongFPSetTestCase.java | 134 + .../util/fingerprint/MemLongFPSet.java | 122 + .../util/iterator/CompositeIterator.java | 111 + .../util/iterator/LineReadingIterator.java | 65 + .../util/iterator/LookaheadIterator.java | 79 + .../util/iterator/RegexpLineIterator.java | 96 + .../iterator/TransformingIteratorWrapper.java | 71 + .../org/archive/util/ms/BlockFileSystem.java | 73 + .../org/archive/util/ms/BlockInputStream.java | 212 ++ .../main/java/org/archive/util/ms/Cp1252.java | 87 + .../util/ms/DefaultBlockFileSystem.java | 287 ++ .../org/archive/util/ms/DefaultEntry.java | 161 + .../main/java/org/archive/util/ms/Doc.java | 161 + .../main/java/org/archive/util/ms/Entry.java | 47 + .../java/org/archive/util/ms/HeaderBlock.java | 97 + .../main/java/org/archive/util/ms/Piece.java | 86 + .../java/org/archive/util/ms/PieceReader.java | 137 + .../java/org/archive/util/ms/PieceTable.java | 284 ++ .../java/org/archive/util/ms/package.html | 114 + commons/src/main/java/org/json/JSONArray.java | 890 +++++ .../src/main/java/org/json/JSONException.java | 28 + .../src/main/java/org/json/JSONObject.java | 1328 ++++++++ .../src/main/java/org/json/JSONString.java | 18 + .../src/main/java/org/json/JSONStringer.java | 78 + .../src/main/java/org/json/JSONTokener.java | 462 +++ .../src/main/java/org/json/JSONWriter.java | 318 ++ commons/src/main/java/org/json/XML.java | 405 +++ .../src/main/java/org/json/XMLTokener.java | 293 ++ .../main/java/st/ata/util/FPGenerator.java | 451 +++ .../main/resources/effective_tld_names.dat | 3030 +++++++++++++++++ .../org/archive/i18n/LocaleCache_en.utf8 | 9 + .../org/archive/i18n/LocaleCache_en_CA.utf8 | 9 + .../i18n/LocaleCache_en_US_borkborkbork.utf8 | 10 + .../org/archive/i18n/LocaleCache_jp.utf8 | 9 + .../resources/org/archive/util/version.txt | 1 + commons/src/site/overview.html | 3 + .../io/BufferedSeekInputStreamTest.java | 71 + .../archive/io/GzippedInputStreamTest.java | 167 + .../archive/io/HeaderedArchiveRecordTest.java | 204 ++ .../archive/io/RecordingInputStreamTest.java | 138 + .../archive/io/RecordingOutputStreamTest.java | 266 ++ .../archive/io/ReplayCharSequenceTest.java | 281 ++ .../io/RepositionableInputStreamTest.java | 76 + .../java/org/archive/io/SinkHandlerTest.java | 84 + .../archive/io/arc/ARCReaderFactoryTest.java | 97 + .../org/archive/io/arc/ARCWriterPoolTest.java | 158 + .../org/archive/io/arc/ARCWriterTest.java | 558 +++ .../org/archive/net/PublicSuffixesTest.java | 113 + .../java/org/archive/net/UURIFactoryTest.java | 1102 ++++++ .../test/java/org/archive/net/UURITest.java | 50 + .../archive/settings/file/BdbModuleTest.java | 99 + .../settings/file/PrefixFinderTest.java | 160 + .../org/archive/surt/SURTTokenizerTest.java | 191 ++ .../org/archive/uid/UUIDGeneratorTest.java | 48 + .../org/archive/util/ArchiveUtilsTest.java | 402 +++ .../org/archive/util/CachedBdbMapTest.java | 89 + .../java/org/archive/util/FileUtilsTest.java | 282 ++ .../java/org/archive/util/IoUtilsTest.java | 49 + .../org/archive/util/MimetypeUtilsTest.java | 69 + .../archive/util/PaddingStringBufferTest.java | 198 ++ .../test/java/org/archive/util/SURTTest.java | 114 + .../org/archive/util/SurtPrefixSetTest.java | 162 + .../java/org/archive/util/TextUtilsTest.java | 130 + .../java/org/archive/util/TransformTest.java | 126 + .../org/archive/util/anvl/ANVLRecordTest.java | 125 + .../fingerprint/ArrayLongFPCacheTest.java | 86 + .../util/fingerprint/LongFPSetCacheTest.java | 96 + .../util/fingerprint/MemLongFPSetTest.java | 76 + .../org/archive/util/ms/15336-doc-preface.doc | Bin 0 -> 28160 bytes .../org/archive/util/ms/15336-doc-preface.txt | Bin 0 -> 16816 bytes .../java/org/archive/util/ms/DocTest.java | 204 ++ .../org/archive/util/ms/PieceReaderTest.java | 156 + .../src/test/java/org/archive/util/ms/X.doc | Bin 0 -> 19456 bytes .../src/test/java/org/archive/util/ms/X.txt | Bin 0 -> 4 bytes .../settings/path/anonymous.resolved.txt | 257 ++ .../org/archive/settings/path/global.get.txt | 23 + .../archive/settings/path/global.resolved.txt | 163 + .../org/archive/settings/path/o1.get.txt | 12 + .../org/archive/settings/path/o1.resolved.txt | 210 ++ dist/HOWTO-Launch-Heritrix.txt | 302 ++ dist/LICENSE.txt | 516 +++ dist/README.txt | 66 + dist/pom.xml | 149 + dist/src/main/assembly/dist.xml | 51 + dist/src/main/assembly/src.xml | 19 + dist/src/main/bin/arcreader | 33 + dist/src/main/bin/arcreader.cmd | 35 + dist/src/main/bin/dependencies.xsl | 35 + dist/src/main/bin/extractor | 35 + dist/src/main/bin/extractor.cmd | 29 + dist/src/main/bin/foreground_heritrix | 39 + dist/src/main/bin/foreground_heritrix.cmd | 40 + dist/src/main/bin/heritrix | 176 + dist/src/main/bin/heritrix.cmd | 296 ++ dist/src/main/bin/hoppath.pl | 124 + dist/src/main/bin/htmlextractor | 34 + dist/src/main/bin/htmlextractor.cmd | 29 + dist/src/main/bin/make_reports.pl | 186 + dist/src/main/bin/manifest_bundle.pl | 104 + dist/src/main/bin/xdocToTxt.xsl | 70 + dist/src/main/conf/heritrix.cacerts | Bin 0 -> 21653 bytes dist/src/main/conf/jndi.properties | 22 + .../conf/jobs/profile-defaults/defaults.xml | 165 + .../profile-crawler-beans.cxml | 50 + dist/src/main/conf/logging.properties | 117 + dist/src/main/extras/pagerank/GenGraph.java | 149 + dist/src/main/extras/pagerank/PageRank.java | 170 + dist/src/main/extras/pagerank/README.txt | 9 + .../main/extras/pagerank/assignUrlIndex.pl | 17 + .../src/main/extras/pagerank/pageRankSetup.pl | 14 + dist/src/main/extras/pagerank/run-pr.sh | 19 + dist/src/main/licenses/ant.LICENSE | 203 ++ dist/src/main/licenses/ant.NOTICE | 20 + dist/src/main/licenses/bsh.LICENSE | 34 + dist/src/main/licenses/commons-cli.LICENSE | 201 ++ dist/src/main/licenses/commons-cli.NOTICE | 5 + dist/src/main/licenses/commons-codec.LICENSE | 201 ++ dist/src/main/licenses/commons-codec.NOTICE | 5 + dist/src/main/licenses/commons-el.LICENSE | 201 ++ dist/src/main/licenses/commons-el.NOTICE | 5 + dist/src/main/licenses/commons-io.LICENSE | 201 ++ dist/src/main/licenses/commons-io.NOTICE | 5 + dist/src/main/licenses/commons-lang.LICENSE | 201 ++ dist/src/main/licenses/commons-lang.NOTICE | 5 + .../src/main/licenses/commons-logging.LICENSE | 201 ++ dist/src/main/licenses/commons-net.LICENSE | 201 ++ dist/src/main/licenses/commons-net.NOTICE | 5 + dist/src/main/licenses/dnsjava-2.0.3.README | 124 + dist/src/main/licenses/fastutil-5.0.7.LICENSE | 165 + dist/src/main/licenses/itext.LICENSE | 24 + dist/src/main/licenses/jasper.LICENSE | 204 ++ dist/src/main/licenses/javaswf.LICENSE | 14 + dist/src/main/licenses/je-3.2.44.LICENSE | 75 + .../main/licenses/jericho-html-2.3.LICENSE | 16 + dist/src/main/licenses/jets3t-0.5.0.LICENSE | 201 ++ dist/src/main/licenses/jetty.LICENSE | 201 ++ dist/src/main/licenses/junit.LICENSE.HTML | 125 + dist/src/main/licenses/libidn.LICENSE | 165 + dist/src/main/licenses/mg4j-1.0.1.LICENSE | 165 + dist/src/main/licenses/oro-2.0.8.LICENSE | 53 + dist/src/main/licenses/poi.LICENSE | 201 ++ dist/src/main/licenses/poi.NOTICE | 35 + dist/src/main/licenses/servlet-4.1.34.LICENSE | 204 ++ .../org/archive/crawler/BasicProfileTest.java | 76 + engine/.classpath | 36 + engine/.cvsignore | 13 + engine/.project | 14 + engine/pom.xml | 150 + engine/src/design/credentials.gif | Bin 0 -> 18691 bytes engine/src/design/credentials.zargo | Bin 0 -> 11239 bytes engine/src/main/assembly/dist.xml | 25 + engine/src/main/java/META-INF/MANIFEST-MF | 0 .../java/org/archive/crawler/Heritrix.java | 437 +++ .../datamodel/CoreAttributeConstants.java | 126 + .../archive/crawler/datamodel/CrawlURI.java | 1775 ++++++++++ .../datamodel/SchedulingConstants.java | 59 + .../crawler/datamodel/UriUniqFilter.java | 148 + .../ClassKeyMatchesRegExpDecideRule.java | 61 + .../archive/crawler/deciderules/package.html | 74 + .../crawler/doc-files/processing_steps.dia | Bin 0 -> 1405 bytes .../crawler/doc-files/processing_steps.png | Bin 0 -> 15555 bytes .../crawler/event/CrawlStateEvent.java | 42 + .../event/CrawlURIDispositionEvent.java | 47 + .../crawler/event/StatSnapshotEvent.java | 46 + .../crawler/framework/Checkpointer.java | 342 ++ .../crawler/framework/CrawlController.java | 655 ++++ .../archive/crawler/framework/CrawlJob.java | 716 ++++ .../crawler/framework/CrawlLimitEnforcer.java | 102 + .../crawler/framework/CrawlStatus.java | 87 + .../org/archive/crawler/framework/Engine.java | 259 ++ .../archive/crawler/framework/Frontier.java | 510 +++ .../org/archive/crawler/framework/Scoper.java | 161 + .../archive/crawler/framework/ToePool.java | 335 ++ .../archive/crawler/framework/ToeThread.java | 663 ++++ .../crawler/frontier/AbstractFrontier.java | 1645 +++++++++ .../AntiCalendarCostAssignmentPolicy.java | 65 + .../archive/crawler/frontier/BdbFrontier.java | 336 ++ .../frontier/BdbMultipleWorkQueues.java | 561 +++ .../crawler/frontier/BdbWorkQueue.java | 175 + .../frontier/BucketQueueAssignmentPolicy.java | 81 + .../frontier/CostAssignmentPolicy.java | 40 + .../crawler/frontier/FrontierJournal.java | 356 ++ .../HostnameQueueAssignmentPolicy.java | 96 + .../frontier/IPQueueAssignmentPolicy.java | 63 + .../frontier/QueueAssignmentPolicy.java | 60 + .../frontier/RecyclingSerialBinding.java | 112 + .../SurtAuthorityQueueAssignmentPolicy.java | 99 + ...mostAssignedSurtQueueAssignmentPolicy.java | 47 + .../frontier/UnitCostAssignmentPolicy.java | 44 + .../frontier/WagCostAssignmentPolicy.java | 71 + .../archive/crawler/frontier/WorkQueue.java | 678 ++++ .../crawler/frontier/WorkQueueFrontier.java | 1718 ++++++++++ .../frontier/ZeroCostAssignmentPolicy.java | 27 + .../precedence/BaseQueuePrecedencePolicy.java | 106 + .../precedence/BaseUriPrecedencePolicy.java | 66 + .../precedence/CostUriPrecedencePolicy.java | 44 + .../HighestUriQueuePrecedencePolicy.java | 130 + .../precedence/HopsUriPrecedencePolicy.java | 58 + .../frontier/precedence/PrecedenceLoader.java | 163 + .../precedence/PrecedenceProvider.java | 84 + .../PreloadedUriPrecedencePolicy.java | 158 + .../precedence/QueuePrecedencePolicy.java | 52 + .../precedence/SimplePrecedenceProvider.java | 55 + .../SuccessCountsQueuePrecedencePolicy.java | 112 + .../precedence/UriPrecedencePolicy.java | 51 + .../crawler/io/NonFatalErrorFormatter.java | 53 + .../crawler/io/RuntimeErrorFormatter.java | 64 + .../crawler/io/StatisticsLogFormatter.java | 39 + .../archive/crawler/io/UriErrorFormatter.java | 59 + .../crawler/io/UriProcessingFormatter.java | 149 + .../java/org/archive/crawler/package.html | 162 + .../postprocessor/CrawlStateUpdater.java | 117 + .../postprocessor/FrontierScheduler.java | 107 + .../crawler/postprocessor/LinksScoper.java | 306 ++ .../postprocessor/LowDiskPauseProcessor.java | 178 + .../SupplementaryLinksScoper.java | 142 + .../prefetch/PreconditionEnforcer.java | 529 +++ .../archive/crawler/prefetch/Preselector.java | 165 + .../crawler/prefetch/QuotaEnforcer.java | 400 +++ .../prefetch/RuntimeLimitEnforcer.java | 200 ++ .../crawler/processor/CrawlMapper.java | 347 ++ .../crawler/processor/HashCrawlMapper.java | 139 + .../crawler/processor/LexicalCrawlMapper.java | 199 ++ .../crawler/reporting/AlertHandler.java | 78 + .../crawler/reporting/AlertThreadGroup.java | 110 + .../crawler/reporting/CrawlStatSnapshot.java | 155 + .../crawler/reporting/CrawlSummaryReport.java | 82 + .../reporting/CrawlerLoggerModule.java | 442 +++ .../crawler/reporting/FrontierReport.java | 21 + .../crawler/reporting/HostsReport.java | 85 + .../crawler/reporting/MimetypesReport.java | 56 + .../crawler/reporting/ProcessorsReport.java | 42 + .../org/archive/crawler/reporting/Report.java | 48 + .../crawler/reporting/ResponseCodeReport.java | 55 + .../archive/crawler/reporting/SeedRecord.java | 116 + .../crawler/reporting/SeedsReport.java | 68 + .../crawler/reporting/SourceTagsReport.java | 65 + .../crawler/reporting/StatisticsTracker.java | 942 +++++ .../crawler/restlet/EditRepresentation.java | 81 + .../crawler/restlet/EngineApplication.java | 90 + .../crawler/restlet/EngineResource.java | 116 + .../archive/crawler/restlet/EnhDirectory.java | 91 + .../crawler/restlet/EnhDirectoryResource.java | 140 + .../org/archive/crawler/restlet/Flash.java | 135 + .../archive/crawler/restlet/JobResource.java | 364 ++ .../crawler/restlet/PagedRepresentation.java | 269 ++ .../crawler/restlet/RateLimitGuard.java | 71 + .../crawler/spring/SheetForDecideRuled.java | 55 + .../crawler/spring/SheetForSurtPrefixes.java | 42 + .../crawler/spring/SheetOverlaysManager.java | 162 + .../crawler/util/BdbUriUniqFilter.java | 386 +++ .../crawler/util/BenchmarkUriUniqFilters.java | 129 + .../crawler/util/BloomUriUniqFilter.java | 151 + .../archive/crawler/util/CheckpointUtils.java | 126 + .../crawler/util/CrawledBytesHistotable.java | 73 + .../util/DiskFPMergeUriUniqFilter.java | 212 ++ .../crawler/util/FPMergeUriUniqFilter.java | 364 ++ .../archive/crawler/util/FPUriUniqFilter.java | 83 + .../org/archive/crawler/util/LogReader.java | 881 +++++ .../org/archive/crawler/util/LogUtils.java | 84 + .../java/org/archive/crawler/util/Logs.java | 52 + .../crawler/util/MemFPMergeUriUniqFilter.java | 69 + .../crawler/util/MemUriUniqFilter.java | 57 + .../crawler/util/NoopUriUniqFilter.java | 55 + .../crawler/util/RecoveryLogMapper.java | 324 ++ .../util/SeedUrlNotFoundException.java | 37 + .../crawler/util/SetBasedUriUniqFilter.java | 141 + .../java/org/archive/crawler/util/Sorts.java | 49 + .../archive/crawler/util/StringIntPair.java | 63 + .../crawler/util/StringIntPairComparator.java | 48 + .../org/archive/crawler/util/TopNSet.java | 95 + .../extractor/CharSequenceLinkExtractor.java | 195 ++ .../extractor/CharSequenceProvider.java | 40 + .../extractor/ExtractErrorListener.java | 47 + .../org/archive/extractor/LinkExtractor.java | 89 + .../extractor/RegexpCSSLinkExtractor.java | 117 + .../extractor/RegexpHTMLLinkExtractor.java | 466 +++ .../extractor/RegexpJSLinkExtractor.java | 114 + .../java/org/archive/extractor/overview.html | 14 + .../src/main/java/org/archive/overview.html | 16 + .../src/main/resources/arcMetaheaderBody.xsl | 167 + .../adaptive/AdaptiveRevisitFrontier_en.utf8 | 83 + .../extras/adaptive/WaitEvaluator_en.utf8 | 48 + .../framework/CrawlControllerImpl_en.utf8 | 104 + .../crawler/framework/CrawlScope_en.utf8 | 19 + .../framework/CrawlerProcessor_en.utf8 | 2 + .../archive/crawler/framework/Scoper_en.utf8 | 15 + .../crawler/frontier/AbstractFrontier_en.utf8 | 109 + .../crawler/frontier/BdbFrontier_en.utf8 | 11 + .../BucketQueueAssignmentPolicy_en.utf8 | 10 + .../frontier/WorkQueueFrontier_en.utf8 | 43 + .../BaseQueuePrecedencePolicy_en.utf8 | 9 + .../BaseUriPrecedencePolicy_en.utf8 | 10 + .../CostUriPrecedencePolicy_en.utf8 | 6 + .../HighestUriQueuePrecedencePolicy_en.utf8 | 5 + .../HopsUriPrecedencePolicy_en.utf8 | 9 + ...SuccessCountsQueuePrecedencePolicy_en.utf8 | 17 + .../postprocessor/CrawlStateUpdater_en.utf8 | 10 + .../postprocessor/FrontierScheduler_en.utf8 | 11 + .../crawler/postprocessor/LinksScoper_en.utf8 | 25 + .../LowDiskPauseProcessor_en.utf8 | 19 + .../SupplementaryLinksScoper_en.utf8 | 12 + .../prefetch/PreconditionEnforcer_en.utf8 | 35 + .../crawler/prefetch/Preselector_en.utf8 | 25 + .../crawler/prefetch/QuotaEnforcer_en.utf8 | 77 + .../prefetch/RuntimeLimitEnforcer_en.utf8 | 41 + .../crawler/processor/CrawlMapper_en.utf8 | 27 + .../crawler/processor/HashCrawlMapper_en.utf8 | 24 + .../processor/LexicalCrawlMapper_en.utf8 | 27 + .../crawler/datamodel/CandidateURITest.java | 69 + .../crawler/datamodel/CrawlURITest.java | 116 + .../framework/CrawlControllerTest.java | 90 + .../framework/CrawlerProcessorTestBase.java | 57 + .../crawler/frontier/BdbFrontierTest.java | 52 + .../frontier/BdbMultipleWorkQueuesTest.java | 109 + .../BucketQueueAssignmentPolicyTest.java | 38 + .../crawler/frontier/FrontierJournalTest.java | 64 + .../BaseQueuePrecedencePolicyTest.java | 39 + .../BaseUriPrecedencePolicyTest.java | 40 + .../CostUriPrecedencePolicyTest.java | 40 + .../HighestUriQueuePrecedencePolicyTest.java | 40 + .../HopsUriPrecedencePolicyTest.java | 38 + ...uccessCountsQueuePrecedencePolicyTest.java | 38 + .../postprocessor/CrawlStateUpdaterTest.java | 41 + .../postprocessor/FrontierSchedulerTest.java | 41 + .../postprocessor/LinksScoperTest.java | 41 + .../LowDiskPauseProcessorTest.java | 41 + .../SupplementaryLinksScoperTest.java | 42 + .../prefetch/PreconditionEnforcerTest.java | 41 + .../crawler/prefetch/PreselectorTest.java | 41 + .../crawler/prefetch/QuotaEnforcerTest.java | 41 + .../prefetch/RuntimeLimitEnforcerTest.java | 41 + .../processor/HashCrawlMapperTest.java | 41 + .../processor/LexicalCrawlMapperTest.java | 41 + ...BackgroundImageExtractionSelfTestCase.java | 98 + .../BadURIsStopPageParsingSelfTest.java | 73 + .../crawler/selftest/CharsetSelfTest.java | 51 + .../crawler/selftest/CheckpointSelfTest.java | 216 ++ .../crawler/selftest/FormAuthSelfTest.java | 118 + .../crawler/selftest/FormAuthServlet.java | 57 + .../crawler/selftest/FramesSelfTestCase.java | 59 + .../crawler/selftest/HttpAuthSelfTest.java | 131 + .../crawler/selftest/KeyWordProcessor.java | 95 + .../selftest/KeyWordUriPrecedencePolicy.java | 41 + .../crawler/selftest/MaxLinkHopsSelfTest.java | 64 + .../crawler/selftest/Precedence1SelfTest.java | 218 ++ .../crawler/selftest/Precedence2SelfTest.java | 51 + .../crawler/selftest/Precedence3SelfTest.java | 193 ++ .../crawler/selftest/Precedence4SelfTest.java | 115 + .../crawler/selftest/RandomServlet.java | 255 ++ .../crawler/selftest/RandomServletTest.java | 50 + .../crawler/selftest/SelfTestBase.java | 369 ++ .../crawler/selftest/SimpleSelfTest.java | 52 + .../crawler/selftest/UserAgentSelfTest.java | 91 + .../crawler/selftest/UserAgentServlet.java | 63 + .../org/archive/crawler/selftest/package.html | 35 + .../crawler/util/BdbUriUniqFilterTest.java | 210 ++ .../crawler/util/BloomUriUniqFilterTest.java | 127 + .../crawler/util/FPUriUniqFilterTest.java | 131 + .../selftest/Auth/basic/basic-loggedin.html | 16 + .../webapps/selftest/Auth/basic/index.html | 24 + .../webapps/selftest/Auth/form/get/error.html | 17 + .../selftest/Auth/form/get/get-loggedin.html | 16 + .../webapps/selftest/Auth/form/get/index.html | 21 + .../selftest/Auth/form/get/success.jsp | 62 + .../src/webapps/selftest/Auth/form/index.html | 23 + .../selftest/Auth/form/post/error.html | 17 + .../selftest/Auth/form/post/index.html | 21 + .../Auth/form/post/post-loggedin.html | 16 + .../selftest/Auth/form/post/success.jsp | 62 + engine/src/webapps/selftest/Auth/index.html | 22 + .../example-background-image.jpeg | Bin 0 -> 6413 bytes .../BackgroundImageExtraction/index.html | 11 + .../BadURIsStopPageParsing/goodone.html | 12 + .../BadURIsStopPageParsing/goodthree.html | 12 + .../BadURIsStopPageParsing/goodtwo.html | 12 + .../BadURIsStopPageParsing/index.html | 24 + .../selftest/BadURIsStopPageParsing/one.html | 14 + .../BadURIsStopPageParsing/three.html | 14 + .../selftest/BadURIsStopPageParsing/two.html | 16 + .../selftest/Charset/charsetselftest_end.html | 12 + .../src/webapps/selftest/Charset/index.html | 13 + .../src/webapps/selftest/Charset/shiftjis.jsp | 33 + engine/src/webapps/selftest/Charset/utf8.jsp | 302 ++ .../webapps/selftest/Checkpoint/index.html | 11 + .../webapps/selftest/FlashParse/index.html | 32 + .../webapps/selftest/FlashParse/pirates.swf | Bin 0 -> 18954 bytes .../webapps/selftest/FlashParse/success.html | 1 + .../selftest/FormTagExtraction/index.html | 33 + .../selftest/FormTagExtraction/inputtag.html | 10 + .../selftest/FormTagExtraction/optiontag.html | 8 + engine/src/webapps/selftest/Frames/index.html | 27 + .../webapps/selftest/Frames/leftframe.html | 10 + .../src/webapps/selftest/Frames/noframe.html | 11 + .../webapps/selftest/Frames/rightframe.html | 10 + .../src/webapps/selftest/Frames/topframe.html | 10 + .../src/webapps/selftest/MaxLinkHops/1.html | 13 + .../src/webapps/selftest/MaxLinkHops/2.html | 12 + .../src/webapps/selftest/MaxLinkHops/3.html | 13 + .../src/webapps/selftest/MaxLinkHops/4.html | 12 + .../src/webapps/selftest/MaxLinkHops/5.html | 12 + .../src/webapps/selftest/MaxLinkHops/6.html | 12 + .../webapps/selftest/MaxLinkHops/index.html | 15 + .../src/webapps/selftest/Refresh/index.html | 10 + .../src/webapps/selftest/Refresh/refresh.html | 11 + .../webapps/selftest/Refresh/refresh2.html | 8 + .../selftest/RobotsExclusion/README.txt | 2 + .../selftest/RobotsExclusion/excluded.html | 1 + .../excluded/level2/level3/excluded.html | 9 + .../selftest/RobotsExclusion/included.html | 10 + .../selftest/RobotsExclusion/index.html | 20 + .../SimpleDocumentTypes/binaries/avi.avi | Bin 0 -> 82944 bytes .../SimpleDocumentTypes/binaries/doc.doc | Bin 0 -> 87552 bytes .../SimpleDocumentTypes/binaries/jpg.jpg | Bin 0 -> 8095 bytes .../SimpleDocumentTypes/binaries/mp3.mp3 | Bin 0 -> 3060 bytes .../SimpleDocumentTypes/binaries/pdf.pdf | Bin 0 -> 3627 bytes .../SimpleDocumentTypes/binaries/ppt.ppt | Bin 0 -> 21504 bytes .../SimpleDocumentTypes/binaries/ps.ps | 7 + .../SimpleDocumentTypes/binaries/rtf.rtf | 1 + .../SimpleDocumentTypes/binaries/wav.wav | Bin 0 -> 21260 bytes .../SimpleDocumentTypes/binaries/wpd.wpd | Bin 0 -> 4197 bytes .../SimpleDocumentTypes/binaries/xls.xls | Bin 0 -> 8704 bytes .../selftest/SimpleDocumentTypes/html.html | 9 + .../selftest/SimpleDocumentTypes/index.html | 27 + .../selftest/SimpleDocumentTypes/txt.txt | 1 + .../SimpleJavascriptExtraction/index.html | 31 + .../jscriptOpenWindow.html | 9 + .../jscriptOpenWindowArg.html | 10 + .../selftest/SpacesInHrefPath/index.html | 10 + .../SpacesInHrefPath/spaces in path.html | 5 + .../anothersub/reluptarget.html | 1 + .../selftest/TrickyRelativeURIs/index.html | 17 + .../TrickyRelativeURIs/reluptricky.html | 1 + engine/src/webapps/selftest/WEB-INF/web.xml | 33 + engine/src/webapps/selftest/index.jsp | 81 + engine/src/webapps/selftest/robots.txt | 4 + engine/src/xsd/arc/1.0/arc.html | 1762 ++++++++++ engine/src/xsd/arc/1.0/arc.xsd | 92 + engine/src/xsd/arc/1.0/example.xml | 28 + .../htdocs/example-background-image.jpeg | Bin 0 -> 6413 bytes .../htdocs/index.html | 11 + .../htdocs/goodone.html | 12 + .../htdocs/goodthree.html | 12 + .../htdocs/goodtwo.html | 12 + .../htdocs/index.html | 24 + .../htdocs/one.html | 14 + .../htdocs/three.html | 14 + .../htdocs/two.html | 16 + .../CharsetSelfTest/htdocs/index.html | 10 + .../selftest/CharsetSelfTest/htdocs/link.html | 0 .../CheckpointSelfTest/profile/config.txt | 0 .../CheckpointSelfTest/profile/seeds.txt | 10 + .../profile/sheets/default.single | 163 + .../FlashParseSelfTest/htdocs/index.html | 32 + .../FlashParseSelfTest/htdocs/pirates.swf | Bin 0 -> 18954 bytes .../FlashParseSelfTest/htdocs/success.html | 1 + .../FormAuthSelfTest/htdocs/failure.html | 9 + .../FormAuthSelfTest/htdocs/index.html | 9 + .../FormAuthSelfTest/htdocs/link1.html | 7 + .../FormAuthSelfTest/htdocs/link2.html | 7 + .../FormAuthSelfTest/htdocs/link3.html | 7 + .../FormAuthSelfTest/htdocs/success.html | 9 + .../FramesSelfTestCase/htdocs/index.html | 27 + .../FramesSelfTestCase/htdocs/leftframe.html | 10 + .../FramesSelfTestCase/htdocs/noframe.html | 11 + .../FramesSelfTestCase/htdocs/rightframe.html | 10 + .../FramesSelfTestCase/htdocs/topframe.html | 10 + .../HttpAuthSelfTest/htdocs/basic/index.html | 7 + .../HttpAuthSelfTest/htdocs/basic/link1.html | 7 + .../HttpAuthSelfTest/htdocs/basic/link2.html | 7 + .../HttpAuthSelfTest/htdocs/basic/link3.html | 7 + .../HttpAuthSelfTest/htdocs/failure.html | 9 + .../HttpAuthSelfTest/htdocs/index.html | 9 + .../HttpAuthSelfTest/htdocs/link1.html | 7 + .../HttpAuthSelfTest/htdocs/link2.html | 7 + .../HttpAuthSelfTest/htdocs/link3.html | 7 + .../HttpAuthSelfTest/htdocs/success.html | 9 + .../MaxLinkHopsSelfTest/htdocs/1.html | 1 + .../MaxLinkHopsSelfTest/htdocs/2.html | 1 + .../MaxLinkHopsSelfTest/htdocs/3.html | 1 + .../MaxLinkHopsSelfTest/htdocs/4.html | 1 + .../MaxLinkHopsSelfTest/htdocs/5.html | 1 + .../MaxLinkHopsSelfTest/htdocs/index.html | 1 + .../Precedence1SelfTest/htdocs/five/a.html | 4 + .../Precedence1SelfTest/htdocs/five/b.html | 4 + .../Precedence1SelfTest/htdocs/one/a.html | 4 + .../Precedence1SelfTest/htdocs/one/b.html | 4 + .../Precedence1SelfTest/htdocs/seed.html | 4 + .../Precedence1SelfTest/htdocs/ten/a.html | 4 + .../Precedence1SelfTest/htdocs/ten/b.html | 4 + .../Precedence2SelfTest/htdocs/five/a.html | 4 + .../Precedence2SelfTest/htdocs/five/b.html | 4 + .../Precedence2SelfTest/htdocs/one/a.html | 4 + .../Precedence2SelfTest/htdocs/one/b.html | 4 + .../Precedence2SelfTest/htdocs/seed.html | 4 + .../Precedence2SelfTest/htdocs/ten/a.html | 4 + .../Precedence2SelfTest/htdocs/ten/b.html | 4 + .../Precedence2SelfTest/profile/rank.txt | 10 + .../Precedence3SelfTest/htdocs/A.html | 3 + .../Precedence3SelfTest/htdocs/B.html | 4 + .../Precedence3SelfTest/htdocs/C.html | 1 + .../Precedence3SelfTest/htdocs/D.html | 4 + .../Precedence3SelfTest/htdocs/E.html | 3 + .../Precedence3SelfTest/htdocs/F.html | 3 + .../Precedence3SelfTest/htdocs/G.html | 2 + .../Precedence3SelfTest/htdocs/H.html | 4 + .../Precedence3SelfTest/htdocs/I.html | 3 + .../Precedence3SelfTest/htdocs/J.html | 4 + .../Precedence3SelfTest/htdocs/K.html | 2 + .../Precedence3SelfTest/htdocs/L.html | 2 + .../Precedence3SelfTest/htdocs/M.html | 3 + .../Precedence3SelfTest/htdocs/N.html | 3 + .../Precedence3SelfTest/htdocs/O.html | 2 + .../Precedence4SelfTest/htdocs/five/a.html | 4 + .../Precedence4SelfTest/htdocs/five/b.html | 4 + .../Precedence4SelfTest/htdocs/one/a.html | 4 + .../Precedence4SelfTest/htdocs/one/b.html | 4 + .../Precedence4SelfTest/htdocs/seed.html | 4 + .../Precedence4SelfTest/htdocs/ten/a.html | 4 + .../Precedence4SelfTest/htdocs/ten/b.html | 4 + .../selftest/SimpleSelfTest/htdocs/index.html | 7 + .../selftest/SimpleSelfTest/htdocs/link1.html | 7 + .../selftest/SimpleSelfTest/htdocs/link2.html | 7 + .../selftest/SimpleSelfTest/htdocs/link3.html | 7 + .../selftest/conf/heritrix.properties | 15 + .../selftest/conf/selftest-crawler-beans.cxml | 181 + modules/.classpath | 28 + modules/.project | 15 + modules/.settings/org.eclipse.jdt.core.prefs | 5 + modules/pom.xml | 78 + .../archive/modules/BeanShellProcessor.java | 176 + .../org/archive/modules/CrawlMetadata.java | 191 ++ .../archive/modules/DefaultProcessorURI.java | 410 +++ .../modules/ModuleAttributeConstants.java | 40 + .../org/archive/modules/PostProcessor.java | 14 + .../org/archive/modules/ProcessResult.java | 67 + .../org/archive/modules/ProcessStatus.java | 63 + .../java/org/archive/modules/Processor.java | 261 ++ .../org/archive/modules/ProcessorChain.java | 106 + .../org/archive/modules/ProcessorLevel.java | 62 + .../archive/modules/ProcessorTestBase.java | 40 + .../org/archive/modules/ProcessorURI.java | 143 + .../modules/canonicalize/BaseRule.java | 90 + .../canonicalize/CanonicalizationRule.java | 54 + .../canonicalize/FixupQueryString.java | 79 + .../modules/canonicalize/LowercaseRule.java | 48 + .../modules/canonicalize/RegexRule.java | 90 + .../RulesCanonicalizationPolicy.java | 99 + .../canonicalize/StripExtraSlashes.java | 48 + .../canonicalize/StripSessionCFIDs.java | 68 + .../modules/canonicalize/StripSessionIDs.java | 82 + .../canonicalize/StripUserinfoRule.java | 55 + .../modules/canonicalize/StripWWWNRule.java | 62 + .../modules/canonicalize/StripWWWRule.java | 58 + .../UriCanonicalizationPolicy.java | 11 + .../modules/credential/Credential.java | 241 ++ .../modules/credential/CredentialAvatar.java | 220 ++ .../modules/credential/CredentialStore.java | 213 ++ .../credential/HtmlFormCredential.java | 182 + .../modules/credential/Rfc2617Credential.java | 179 + .../archive/modules/credential/package.html | 16 + .../modules/deciderules/AcceptDecideRule.java | 44 + .../AddRedirectFromRootServerToScope.java | 73 + .../deciderules/BeanShellDecideRule.java | 173 + .../deciderules/ContentLengthDecideRule.java | 61 + .../ContentTypeMatchesRegExpDecideRule.java | 43 + ...ContentTypeNotMatchesRegExpDecideRule.java | 51 + .../modules/deciderules/DecideResult.java | 54 + .../modules/deciderules/DecideRule.java | 85 + .../deciderules/DecideRuleSequence.java | 63 + ...eedsDocumentLengthThresholdDecideRule.java | 14 + .../ExternalGeoLocationDecideRule.java | 131 + .../ExternalGeoLookupInterface.java | 39 + .../deciderules/FetchStatusDecideRule.java | 69 + .../FetchStatusMatchesRegExpDecideRule.java | 45 + ...FetchStatusNotMatchesRegExpDecideRule.java | 52 + .../modules/deciderules/HasViaDecideRule.java | 54 + .../HopsPathMatchesRegExpDecideRule.java | 50 + .../MatchesFilePatternDecideRule.java | 107 + .../MatchesListRegExpDecideRule.java | 132 + .../deciderules/MatchesRegExpDecideRule.java | 74 + ...ceedsDocumentLengthTresholdDecideRule.java | 131 + .../NotMatchesFilePatternDecideRule.java | 61 + .../NotMatchesListRegExpDecideRule.java | 62 + .../NotMatchesRegExpDecideRule.java | 60 + .../PathologicalPathDecideRule.java | 100 + .../deciderules/PredicatedDecideRule.java | 60 + .../PrerequisiteAcceptDecideRule.java | 56 + .../modules/deciderules/RejectDecideRule.java | 44 + .../deciderules/SeedAcceptDecideRule.java | 53 + .../deciderules/TooManyHopsDecideRule.java | 85 + .../TooManyPathSegmentsDecideRule.java | 86 + .../deciderules/TransclusionDecideRule.java | 122 + .../recrawl/IdenticalDigestDecideRule.java | 88 + .../surt/NotOnDomainsDecideRule.java | 61 + .../surt/NotOnHostsDecideRule.java | 60 + .../surt/NotSurtPrefixedDecideRule.java | 63 + .../deciderules/surt/OnDomainsDecideRule.java | 62 + .../deciderules/surt/OnHostsDecideRule.java | 64 + .../surt/ScopePlusOneDecideRule.java | 155 + .../surt/SurtPrefixedDecideRule.java | 306 ++ .../extractor/AggressiveExtractorHTML.java | 72 + .../modules/extractor/ContentExtractor.java | 117 + .../extractor/ContentExtractorTestBase.java | 147 + .../modules/extractor/CustomSWFTags.java | 58 + .../archive/modules/extractor/Extractor.java | 131 + .../modules/extractor/ExtractorCSS.java | 179 + .../modules/extractor/ExtractorDOC.java | 141 + .../modules/extractor/ExtractorHTML.java | 865 +++++ .../modules/extractor/ExtractorHTTP.java | 95 + .../extractor/ExtractorImpliedURI.java | 203 ++ .../modules/extractor/ExtractorJS.java | 202 ++ .../modules/extractor/ExtractorPDF.java | 155 + .../extractor/ExtractorParameters.java | 32 + .../modules/extractor/ExtractorSWF.java | 286 ++ .../modules/extractor/ExtractorURI.java | 191 ++ .../modules/extractor/ExtractorUniversal.java | 550 +++ .../modules/extractor/ExtractorXML.java | 157 + .../modules/extractor/HTMLLinkContext.java | 69 + .../modules/extractor/HTTPContentDigest.java | 191 ++ .../org/archive/modules/extractor/Hop.java | 80 + .../extractor/JerichoExtractorHTML.java | 457 +++ .../org/archive/modules/extractor/Link.java | 166 + .../modules/extractor/LinkContext.java | 91 + .../archive/modules/extractor/PDFParser.java | 269 ++ .../extractor/StringExtractorTestBase.java | 78 + .../modules/extractor/TempDirProvider.java | 10 + .../extractor/TrapSuppressExtractor.java | 91 + .../extractor/UriErrorLoggerModule.java | 39 + .../fetcher/AbstractCookieStorage.java | 230 ++ .../modules/fetcher/BdbCookieStorage.java | 116 + .../modules/fetcher/CookieStorage.java | 39 + .../modules/fetcher/DefaultServerCache.java | 216 ++ .../org/archive/modules/fetcher/FetchDNS.java | 351 ++ .../archive/modules/fetcher/FetchErrors.java | 20 + .../org/archive/modules/fetcher/FetchFTP.java | 548 +++ .../archive/modules/fetcher/FetchHTTP.java | 1515 +++++++++ .../archive/modules/fetcher/FetchStats.java | 182 + .../modules/fetcher/FetchStatusCodes.java | 119 + .../HeritrixHttpMethodRetryHandler.java | 88 + .../HeritrixProtocolSocketFactory.java | 200 ++ .../HeritrixSSLProtocolSocketFactory.java | 155 + .../archive/modules/fetcher/HostResolver.java | 33 + .../modules/fetcher/SimpleCookieStorage.java | 53 + .../modules/fetcher/UserAgentProvider.java | 42 + .../archive/modules/net/BdbServerCache.java | 73 + .../org/archive/modules/net/CrawlHost.java | 208 ++ .../org/archive/modules/net/CrawlServer.java | 305 ++ .../modules/net/DefaultTempDirProvider.java | 63 + .../archive/modules/net/RobotsDirectives.java | 69 + .../modules/net/RobotsExclusionPolicy.java | 264 ++ .../modules/net/RobotsHonoringPolicy.java | 191 ++ .../org/archive/modules/net/Robotstxt.java | 165 + .../org/archive/modules/net/ServerCache.java | 49 + .../archive/modules/net/ServerCacheUtil.java | 71 + .../org/archive/modules/package-info.java | 56 + .../recrawl/FetchHistoryProcessor.java | 139 + .../modules/recrawl/PersistLoadProcessor.java | 60 + .../modules/recrawl/PersistLogProcessor.java | 113 + .../recrawl/PersistOnlineProcessor.java | 112 + .../modules/recrawl/PersistProcessor.java | 306 ++ .../recrawl/PersistStoreProcessor.java | 53 + .../recrawl/RecrawlAttributeConstants.java | 51 + .../modules/seeds/SeedFileIterator.java | 130 + .../archive/modules/seeds/SeedListener.java | 37 + .../org/archive/modules/seeds/SeedModule.java | 74 + .../archive/modules/seeds/TextSeedModule.java | 171 + .../modules/writer/ARCWriterProcessor.java | 270 ++ .../archive/modules/writer/Kw3Constants.java | 40 + .../modules/writer/Kw3WriterProcessor.java | 443 +++ .../modules/writer/MirrorWriterProcessor.java | 1734 ++++++++++ .../modules/writer/WARCWriterProcessor.java | 588 ++++ .../modules/writer/WriterPoolProcessor.java | 484 +++ .../modules/BeanShellProcessor_en.utf8 | 22 + .../org/archive/modules/Processor_en.utf8 | 8 + .../modules/canonicalize/BaseRule_en.utf8 | 2 + .../canonicalize/FixupQueryString_en.utf8 | 10 + .../canonicalize/LowercaseRule_en.utf8 | 2 + .../modules/canonicalize/RegexRule_en.utf8 | 15 + .../canonicalize/StripExtraSlashes_en.utf8 | 5 + .../canonicalize/StripSessionCFIDs_en.utf8 | 6 + .../canonicalize/StripSessionIDs_en.utf8 | 7 + .../canonicalize/StripUserinfoRule_en.utf8 | 6 + .../canonicalize/StripWWWNRule_en.utf8 | 12 + .../modules/canonicalize/StripWWWRule_en.utf8 | 9 + .../credential/CredentialStore_en.utf8 | 8 + .../modules/credential/Credential_en.utf8 | 2 + .../credential/HtmlFormCredential_en.utf8 | 17 + .../credential/Rfc2617Credential_en.utf8 | 16 + .../deciderules/DecideRuleSequence_en.utf8 | 6 + .../modules/deciderules/DecideRule_en.utf8 | 9 + .../deciderules/HasViaDecideRule_en.utf8 | 4 + .../HopsPathMatchesRegExpDecideRule_en.utf8 | 4 + .../MatchesRegExpDecideRule_en.utf8 | 2 + .../PredicatedAcceptDecideRule_en.utf8 | 2 + .../extractor/AggressiveExtractorHTML_en.utf8 | 6 + .../modules/extractor/ExtractorCSS_en.utf8 | 2 + .../modules/extractor/ExtractorDOC_en.utf8 | 2 + .../modules/extractor/ExtractorHTML_en.utf8 | 63 + .../modules/extractor/ExtractorHTTP_en.utf8 | 2 + .../extractor/ExtractorImpliedURI_en.utf8 | 19 + .../modules/extractor/ExtractorJS_en.utf8 | 2 + .../modules/extractor/ExtractorPDF_en.utf8 | 12 + .../modules/extractor/ExtractorSWF_en.utf8 | 2 + .../modules/extractor/ExtractorURI_en.utf8 | 3 + .../extractor/ExtractorUniversal_en.utf8 | 14 + .../modules/extractor/ExtractorXML_en.utf8 | 2 + .../modules/extractor/Extractor_en.utf8 | 2 + .../extractor/HTTPContentDigest_en.utf8 | 35 + .../extractor/TrapSuppressExtractor_en.utf8 | 64 + .../archive/modules/fetcher/FetchDNS_en.utf8 | 22 + .../archive/modules/fetcher/FetchFTP_en.utf8 | 38 + .../archive/modules/fetcher/FetchHTTP_en.utf8 | 136 + .../modules/net/RobotsHonoringPolicy_en.utf8 | 30 + .../recrawl/FetchHistoryProcessor_en.utf8 | 8 + .../recrawl/PersistLoadProcessor_en.utf8 | 3 + .../recrawl/PersistLogProcessor_en.utf8 | 7 + .../recrawl/PersistOnlineProcessor_en.utf8 | 5 + .../recrawl/PersistStoreProcessor_en.utf8 | 3 + .../modules/writer/ARCWriterProcessor_en.utf8 | 12 + .../writer/MirrorWriterProcessor_en.utf8 | 87 + .../writer/WARCWriterProcessor_en.utf8 | 30 + .../writer/WriterPoolProcessor_en.utf8 | 62 + .../modules/writer/arc_metadata_template.xml | 14 + .../modules/BeanShellProcessorTest.java | 37 + .../canonicalize/FixupQueryStringTest.java | 58 + .../canonicalize/LowercaseRuleTest.java | 42 + .../modules/canonicalize/RegexRuleTest.java | 75 + .../RulesCanonicalizationPolicyTest.java | 63 + .../canonicalize/StripSessionCFIDsTest.java | 38 + .../canonicalize/StripSessionIDsTest.java | 108 + .../canonicalize/StripUserinfoRuleTest.java | 60 + .../canonicalize/StripWWWNRuleTest.java | 52 + .../canonicalize/StripWWWRuleTest.java | 59 + .../credential/CredentialStoreTest.java | 103 + .../credential/HtmlFormCredentialTest.java | 39 + .../credential/Rfc2617CredentialTest.java | 39 + .../deciderules/DecideRuleSequenceTest.java | 36 + .../deciderules/HasViaDecideRuleTest.java | 37 + .../HopsPathMatchesRegExpDecideRuleTest.java | 37 + .../AggressiveExtractorHTMLTest.java | 42 + .../modules/extractor/ExtractorCSSTest.java | 97 + .../modules/extractor/ExtractorDOCTest.java | 42 + .../modules/extractor/ExtractorHTMLTest.java | 204 ++ .../modules/extractor/ExtractorHTTPTest.java | 39 + .../extractor/ExtractorImpliedURITest.java | 39 + .../modules/extractor/ExtractorJSTest.java | 39 + .../modules/extractor/ExtractorPDFTest.java | 39 + .../modules/extractor/ExtractorSWFTest.java | 39 + .../modules/extractor/ExtractorURITest.java | 39 + .../extractor/ExtractorUniversalTest.java | 39 + .../modules/extractor/ExtractorXMLTest.java | 39 + .../extractor/HTTPContentDigestTest.java | 39 + .../extractor/JerichoExtractorHTMLTest.java | 147 + .../extractor/UnitTestUriLoggerModule.java | 32 + .../archive/modules/fetcher/FetchDNSTest.java | 38 + .../archive/modules/fetcher/FetchFTPTest.java | 38 + .../modules/fetcher/FetchHTTPTest.java | 52 + .../archive/modules/net/CrawlHostTest.java | 46 + .../archive/modules/net/CrawlServerTest.java | 53 + .../modules/net/RobotsHonoringPolicyTest.java | 38 + .../archive/modules/net/RobotstxtTest.java | 108 + .../archive/modules/net/ServerCacheTest.java | 68 + .../recrawl/FetchHistoryProcessorTest.java | 13 + .../recrawl/PersistLoadProcessorTest.java | 13 + .../recrawl/PersistLogProcessorTest.java | 13 + .../recrawl/PersistStoreProcessorTest.java | 13 + .../modules/seeds/SeedFileIteratorTest.java | 83 + .../writer/ARCWriterProcessorTest.java | 64 + .../writer/MirrorWriterProcessorTest.java | 37 + .../writer/WARCWriterProcessorTest.java | 62 + pom.xml | 359 ++ 966 files changed, 124796 insertions(+) create mode 100644 .classpath create mode 100644 .project create mode 100644 commons/.classpath create mode 100644 commons/.project create mode 100644 commons/.settings/org.eclipse.jdt.core.prefs create mode 100644 commons/pom.xml create mode 100644 commons/src/main/java/org/apache/commons/httpclient/Cookie.java create mode 100644 commons/src/main/java/org/apache/commons/httpclient/HttpConnection.java create mode 100644 commons/src/main/java/org/apache/commons/httpclient/HttpMethodBase.java create mode 100644 commons/src/main/java/org/apache/commons/httpclient/HttpParser.java create mode 100644 commons/src/main/java/org/apache/commons/httpclient/HttpState.java create mode 100644 commons/src/main/java/org/apache/commons/httpclient/cookie/CookieSpec.java create mode 100644 commons/src/main/java/org/apache/commons/httpclient/cookie/CookieSpecBase.java create mode 100644 commons/src/main/java/org/apache/commons/httpclient/cookie/IgnoreCookiesSpec.java create mode 100644 commons/src/main/java/org/apache/commons/pool/impl/FairGenericObjectPool.java create mode 100644 commons/src/main/java/org/apache/commons/pool/impl/FairGenericObjectPoolTest.java create mode 100644 commons/src/main/java/org/apache/commons/pool/impl/GenericObjectPool.java create mode 100644 commons/src/main/java/org/archive/bdb/BdbModule.java create mode 100644 commons/src/main/java/org/archive/checkpointing/CheckpointInputStream.java create mode 100644 commons/src/main/java/org/archive/checkpointing/CheckpointRecovery.java create mode 100644 commons/src/main/java/org/archive/checkpointing/Checkpointable.java create mode 100644 commons/src/main/java/org/archive/checkpointing/Checkpointer.java create mode 100644 commons/src/main/java/org/archive/checkpointing/DefaultCheckpointRecovery.java create mode 100644 commons/src/main/java/org/archive/checkpointing/RecoverAction.java create mode 100644 commons/src/main/java/org/archive/httpclient/ConfigurableX509TrustManager.java create mode 100644 commons/src/main/java/org/archive/httpclient/HttpRecorderGetMethod.java create mode 100644 commons/src/main/java/org/archive/httpclient/HttpRecorderMethod.java create mode 100644 commons/src/main/java/org/archive/httpclient/HttpRecorderPostMethod.java create mode 100644 commons/src/main/java/org/archive/httpclient/SingleHttpConnectionManager.java create mode 100644 commons/src/main/java/org/archive/httpclient/ThreadLocalHttpConnectionManager.java create mode 100644 commons/src/main/java/org/archive/httpclient/package.html create mode 100644 commons/src/main/java/org/archive/io/Arc2Warc.java create mode 100644 commons/src/main/java/org/archive/io/ArchiveFileConstants.java create mode 100644 commons/src/main/java/org/archive/io/ArchiveReader.java create mode 100644 commons/src/main/java/org/archive/io/ArchiveReaderFactory.java create mode 100644 commons/src/main/java/org/archive/io/ArchiveRecord.java create mode 100644 commons/src/main/java/org/archive/io/ArchiveRecordHeader.java create mode 100644 commons/src/main/java/org/archive/io/ArraySeekInputStream.java create mode 100644 commons/src/main/java/org/archive/io/BufferedSeekInputStream.java create mode 100644 commons/src/main/java/org/archive/io/ByteReplayCharSequence.java create mode 100644 commons/src/main/java/org/archive/io/CharSubSequence.java create mode 100644 commons/src/main/java/org/archive/io/CompositeFileInputStream.java create mode 100644 commons/src/main/java/org/archive/io/CompositeFileReader.java create mode 100644 commons/src/main/java/org/archive/io/CrawlerJournal.java create mode 100644 commons/src/main/java/org/archive/io/DefaultWriterPoolSettings.java create mode 100644 commons/src/main/java/org/archive/io/Endian.java create mode 100644 commons/src/main/java/org/archive/io/GenerationFileHandler.java create mode 100644 commons/src/main/java/org/archive/io/GzipHeader.java create mode 100644 commons/src/main/java/org/archive/io/GzippedInputStream.java create mode 100644 commons/src/main/java/org/archive/io/HeaderedArchiveRecord.java create mode 100644 commons/src/main/java/org/archive/io/LoudObjectOutputStream.java create mode 100644 commons/src/main/java/org/archive/io/MultiByteReplayCharSequence.java create mode 100644 commons/src/main/java/org/archive/io/NoGzipMagicException.java create mode 100644 commons/src/main/java/org/archive/io/ObjectPlusFilesInputStream.java create mode 100644 commons/src/main/java/org/archive/io/ObjectPlusFilesOutputStream.java create mode 100644 commons/src/main/java/org/archive/io/OriginSeekInputStream.java create mode 100644 commons/src/main/java/org/archive/io/RandomAccessInputStream.java create mode 100644 commons/src/main/java/org/archive/io/RandomAccessOutputStream.java create mode 100644 commons/src/main/java/org/archive/io/ReadSource.java create mode 100644 commons/src/main/java/org/archive/io/RecorderIOException.java create mode 100644 commons/src/main/java/org/archive/io/RecorderLengthExceededException.java create mode 100644 commons/src/main/java/org/archive/io/RecorderTimeoutException.java create mode 100644 commons/src/main/java/org/archive/io/RecorderTooMuchHeaderException.java create mode 100644 commons/src/main/java/org/archive/io/RecordingInputStream.java create mode 100644 commons/src/main/java/org/archive/io/RecordingOutputStream.java create mode 100644 commons/src/main/java/org/archive/io/RecoverableIOException.java create mode 100644 commons/src/main/java/org/archive/io/RecyclingFastBufferedOutputStream.java create mode 100644 commons/src/main/java/org/archive/io/ReplayCharSequence.java create mode 100644 commons/src/main/java/org/archive/io/ReplayInputStream.java create mode 100644 commons/src/main/java/org/archive/io/RepositionableInputStream.java create mode 100644 commons/src/main/java/org/archive/io/SafeSeekInputStream.java create mode 100644 commons/src/main/java/org/archive/io/SeekInputStream.java create mode 100644 commons/src/main/java/org/archive/io/SeekReader.java create mode 100644 commons/src/main/java/org/archive/io/SeekReaderCharSequence.java create mode 100644 commons/src/main/java/org/archive/io/SinkHandlerLogThread.java create mode 100644 commons/src/main/java/org/archive/io/UTF8Bytes.java create mode 100644 commons/src/main/java/org/archive/io/Warc2Arc.java create mode 100644 commons/src/main/java/org/archive/io/WriterPool.java create mode 100644 commons/src/main/java/org/archive/io/WriterPoolMember.java create mode 100644 commons/src/main/java/org/archive/io/WriterPoolSettings.java create mode 100644 commons/src/main/java/org/archive/io/arc/ARC2WCDX.java create mode 100644 commons/src/main/java/org/archive/io/arc/ARCConstants.java create mode 100644 commons/src/main/java/org/archive/io/arc/ARCLocation.java create mode 100644 commons/src/main/java/org/archive/io/arc/ARCReader.java create mode 100644 commons/src/main/java/org/archive/io/arc/ARCReaderFactory.java create mode 100644 commons/src/main/java/org/archive/io/arc/ARCRecord.java create mode 100644 commons/src/main/java/org/archive/io/arc/ARCRecordMetaData.java create mode 100644 commons/src/main/java/org/archive/io/arc/ARCUtils.java create mode 100644 commons/src/main/java/org/archive/io/arc/ARCWriter.java create mode 100644 commons/src/main/java/org/archive/io/arc/ARCWriterPool.java create mode 100644 commons/src/main/java/org/archive/io/arc/package.html create mode 100644 commons/src/main/java/org/archive/io/warc/WARCConstants.java create mode 100644 commons/src/main/java/org/archive/io/warc/WARCReader.java create mode 100644 commons/src/main/java/org/archive/io/warc/WARCReaderFactory.java create mode 100644 commons/src/main/java/org/archive/io/warc/WARCRecord.java create mode 100644 commons/src/main/java/org/archive/io/warc/WARCWriter.java create mode 100644 commons/src/main/java/org/archive/io/warc/WARCWriterPool.java create mode 100644 commons/src/main/java/org/archive/io/warc/WARCWriterTest.java create mode 100644 commons/src/main/java/org/archive/io/warc/package.html create mode 100644 commons/src/main/java/org/archive/net/ClientFTP.java create mode 100644 commons/src/main/java/org/archive/net/DownloadURLConnection.java create mode 100644 commons/src/main/java/org/archive/net/FTPException.java create mode 100644 commons/src/main/java/org/archive/net/LaxURI.java create mode 100644 commons/src/main/java/org/archive/net/LaxURLCodec.java create mode 100644 commons/src/main/java/org/archive/net/PublicSuffixes.java create mode 100644 commons/src/main/java/org/archive/net/UURI.java create mode 100644 commons/src/main/java/org/archive/net/UURIFactory.java create mode 100644 commons/src/main/java/org/archive/net/md5/Handler.java create mode 100644 commons/src/main/java/org/archive/net/md5/Md5URLConnection.java create mode 100644 commons/src/main/java/org/archive/net/rsync/Handler.java create mode 100644 commons/src/main/java/org/archive/net/rsync/RsyncURLConnection.java create mode 100644 commons/src/main/java/org/archive/net/s3/Handler.java create mode 100644 commons/src/main/java/org/archive/net/s3/S3URLConnection.java create mode 100644 commons/src/main/java/org/archive/queue/StoredQueue.java create mode 100644 commons/src/main/java/org/archive/queue/StoredQueueTest.java create mode 100644 commons/src/main/java/org/archive/spring/BeanFieldsPatternValidator.java create mode 100644 commons/src/main/java/org/archive/spring/ConfigFile.java create mode 100644 commons/src/main/java/org/archive/spring/ConfigPath.java create mode 100644 commons/src/main/java/org/archive/spring/ConfigPathConfigurer.java create mode 100644 commons/src/main/java/org/archive/spring/ConfigPathEditor.java create mode 100644 commons/src/main/java/org/archive/spring/ConfigString.java create mode 100644 commons/src/main/java/org/archive/spring/HasKeyedProperties.java create mode 100644 commons/src/main/java/org/archive/spring/HasValidator.java create mode 100644 commons/src/main/java/org/archive/spring/KeyedProperties.java create mode 100644 commons/src/main/java/org/archive/spring/OverlayContext.java create mode 100644 commons/src/main/java/org/archive/spring/OverlayMapsSource.java create mode 100644 commons/src/main/java/org/archive/spring/PathSharingContext.java create mode 100644 commons/src/main/java/org/archive/spring/Sheet.java create mode 100644 commons/src/main/java/org/archive/spring/WriteTarget.java create mode 100644 commons/src/main/java/org/archive/state/ModuleTestBase.java create mode 100644 commons/src/main/java/org/archive/surt/SURTTokenizer.java create mode 100644 commons/src/main/java/org/archive/uid/Generator.java create mode 100644 commons/src/main/java/org/archive/uid/GeneratorFactory.java create mode 100644 commons/src/main/java/org/archive/uid/UUIDGenerator.java create mode 100644 commons/src/main/java/org/archive/uid/package.html create mode 100644 commons/src/main/java/org/archive/util/AbstractLongFPSet.java create mode 100644 commons/src/main/java/org/archive/util/ArchiveUtils.java create mode 100644 commons/src/main/java/org/archive/util/Base32.java create mode 100644 commons/src/main/java/org/archive/util/BenchmarkBlooms.java create mode 100644 commons/src/main/java/org/archive/util/BloomFilter.java create mode 100644 commons/src/main/java/org/archive/util/BloomFilter32bit.java create mode 100644 commons/src/main/java/org/archive/util/BloomFilter32bitSplit.java create mode 100644 commons/src/main/java/org/archive/util/BloomFilter32bp2.java create mode 100644 commons/src/main/java/org/archive/util/BloomFilter32bp2Split.java create mode 100644 commons/src/main/java/org/archive/util/BloomFilter64bit.java create mode 100644 commons/src/main/java/org/archive/util/CachedBdbMap.java create mode 100644 commons/src/main/java/org/archive/util/CollectionUtils.java create mode 100644 commons/src/main/java/org/archive/util/DNSJavaUtil.java create mode 100644 commons/src/main/java/org/archive/util/DevUtils.java create mode 100644 commons/src/main/java/org/archive/util/FileUtils.java create mode 100644 commons/src/main/java/org/archive/util/Histotable.java create mode 100644 commons/src/main/java/org/archive/util/InetAddressUtil.java create mode 100644 commons/src/main/java/org/archive/util/InterruptibleCharSequence.java create mode 100644 commons/src/main/java/org/archive/util/InterruptibleCharSequenceTest.java create mode 100644 commons/src/main/java/org/archive/util/Inverter.java create mode 100644 commons/src/main/java/org/archive/util/IoUtils.java create mode 100644 commons/src/main/java/org/archive/util/JavaLiterals.java create mode 100644 commons/src/main/java/org/archive/util/JndiUtils.java create mode 100644 commons/src/main/java/org/archive/util/LRU.java create mode 100644 commons/src/main/java/org/archive/util/LongWrapper.java create mode 100644 commons/src/main/java/org/archive/util/MimetypeUtils.java create mode 100644 commons/src/main/java/org/archive/util/MultiReporter.java create mode 100644 commons/src/main/java/org/archive/util/OneLineSimpleLogger.java create mode 100644 commons/src/main/java/org/archive/util/PaddingStringBuffer.java create mode 100644 commons/src/main/java/org/archive/util/PatternMatcherRecycler.java create mode 100644 commons/src/main/java/org/archive/util/PreJ15Utils.java create mode 100644 commons/src/main/java/org/archive/util/PrefixFinder.java create mode 100644 commons/src/main/java/org/archive/util/PrefixSet.java create mode 100644 commons/src/main/java/org/archive/util/ProcessUtils.java create mode 100644 commons/src/main/java/org/archive/util/ProgressStatisticsReporter.java create mode 100644 commons/src/main/java/org/archive/util/PropertyUtils.java create mode 100644 commons/src/main/java/org/archive/util/Recorder.java create mode 100644 commons/src/main/java/org/archive/util/RecorderMarker.java create mode 100644 commons/src/main/java/org/archive/util/Reporter.java create mode 100644 commons/src/main/java/org/archive/util/SURT.java create mode 100644 commons/src/main/java/org/archive/util/SubList.java create mode 100644 commons/src/main/java/org/archive/util/SurtPrefixSet.java create mode 100644 commons/src/main/java/org/archive/util/TestUtils.java create mode 100644 commons/src/main/java/org/archive/util/TextUtils.java create mode 100644 commons/src/main/java/org/archive/util/TimestampSerialno.java create mode 100644 commons/src/main/java/org/archive/util/TmpDirTestCase.java create mode 100644 commons/src/main/java/org/archive/util/Transform.java create mode 100644 commons/src/main/java/org/archive/util/Transformer.java create mode 100644 commons/src/main/java/org/archive/util/anvl/ANVLRecord.java create mode 100644 commons/src/main/java/org/archive/util/anvl/ANVLRecords.java create mode 100644 commons/src/main/java/org/archive/util/anvl/Element.java create mode 100644 commons/src/main/java/org/archive/util/anvl/Label.java create mode 100644 commons/src/main/java/org/archive/util/anvl/SubElement.java create mode 100644 commons/src/main/java/org/archive/util/anvl/Value.java create mode 100644 commons/src/main/java/org/archive/util/anvl/package.html create mode 100644 commons/src/main/java/org/archive/util/bdbje/EnhancedEnvironment.java create mode 100644 commons/src/main/java/org/archive/util/fingerprint/ArrayLongFPCache.java create mode 100644 commons/src/main/java/org/archive/util/fingerprint/LongFPSet.java create mode 100644 commons/src/main/java/org/archive/util/fingerprint/LongFPSetCache.java create mode 100644 commons/src/main/java/org/archive/util/fingerprint/LongFPSetTestCase.java create mode 100644 commons/src/main/java/org/archive/util/fingerprint/MemLongFPSet.java create mode 100644 commons/src/main/java/org/archive/util/iterator/CompositeIterator.java create mode 100644 commons/src/main/java/org/archive/util/iterator/LineReadingIterator.java create mode 100644 commons/src/main/java/org/archive/util/iterator/LookaheadIterator.java create mode 100644 commons/src/main/java/org/archive/util/iterator/RegexpLineIterator.java create mode 100644 commons/src/main/java/org/archive/util/iterator/TransformingIteratorWrapper.java create mode 100644 commons/src/main/java/org/archive/util/ms/BlockFileSystem.java create mode 100644 commons/src/main/java/org/archive/util/ms/BlockInputStream.java create mode 100644 commons/src/main/java/org/archive/util/ms/Cp1252.java create mode 100644 commons/src/main/java/org/archive/util/ms/DefaultBlockFileSystem.java create mode 100644 commons/src/main/java/org/archive/util/ms/DefaultEntry.java create mode 100644 commons/src/main/java/org/archive/util/ms/Doc.java create mode 100644 commons/src/main/java/org/archive/util/ms/Entry.java create mode 100644 commons/src/main/java/org/archive/util/ms/HeaderBlock.java create mode 100644 commons/src/main/java/org/archive/util/ms/Piece.java create mode 100644 commons/src/main/java/org/archive/util/ms/PieceReader.java create mode 100644 commons/src/main/java/org/archive/util/ms/PieceTable.java create mode 100644 commons/src/main/java/org/archive/util/ms/package.html create mode 100644 commons/src/main/java/org/json/JSONArray.java create mode 100644 commons/src/main/java/org/json/JSONException.java create mode 100644 commons/src/main/java/org/json/JSONObject.java create mode 100644 commons/src/main/java/org/json/JSONString.java create mode 100644 commons/src/main/java/org/json/JSONStringer.java create mode 100644 commons/src/main/java/org/json/JSONTokener.java create mode 100644 commons/src/main/java/org/json/JSONWriter.java create mode 100644 commons/src/main/java/org/json/XML.java create mode 100644 commons/src/main/java/org/json/XMLTokener.java create mode 100644 commons/src/main/java/st/ata/util/FPGenerator.java create mode 100644 commons/src/main/resources/effective_tld_names.dat create mode 100644 commons/src/main/resources/org/archive/i18n/LocaleCache_en.utf8 create mode 100644 commons/src/main/resources/org/archive/i18n/LocaleCache_en_CA.utf8 create mode 100644 commons/src/main/resources/org/archive/i18n/LocaleCache_en_US_borkborkbork.utf8 create mode 100644 commons/src/main/resources/org/archive/i18n/LocaleCache_jp.utf8 create mode 100644 commons/src/main/resources/org/archive/util/version.txt create mode 100644 commons/src/site/overview.html create mode 100644 commons/src/test/java/org/archive/io/BufferedSeekInputStreamTest.java create mode 100644 commons/src/test/java/org/archive/io/GzippedInputStreamTest.java create mode 100644 commons/src/test/java/org/archive/io/HeaderedArchiveRecordTest.java create mode 100644 commons/src/test/java/org/archive/io/RecordingInputStreamTest.java create mode 100644 commons/src/test/java/org/archive/io/RecordingOutputStreamTest.java create mode 100644 commons/src/test/java/org/archive/io/ReplayCharSequenceTest.java create mode 100644 commons/src/test/java/org/archive/io/RepositionableInputStreamTest.java create mode 100644 commons/src/test/java/org/archive/io/SinkHandlerTest.java create mode 100644 commons/src/test/java/org/archive/io/arc/ARCReaderFactoryTest.java create mode 100644 commons/src/test/java/org/archive/io/arc/ARCWriterPoolTest.java create mode 100644 commons/src/test/java/org/archive/io/arc/ARCWriterTest.java create mode 100644 commons/src/test/java/org/archive/net/PublicSuffixesTest.java create mode 100644 commons/src/test/java/org/archive/net/UURIFactoryTest.java create mode 100644 commons/src/test/java/org/archive/net/UURITest.java create mode 100644 commons/src/test/java/org/archive/settings/file/BdbModuleTest.java create mode 100644 commons/src/test/java/org/archive/settings/file/PrefixFinderTest.java create mode 100644 commons/src/test/java/org/archive/surt/SURTTokenizerTest.java create mode 100644 commons/src/test/java/org/archive/uid/UUIDGeneratorTest.java create mode 100644 commons/src/test/java/org/archive/util/ArchiveUtilsTest.java create mode 100644 commons/src/test/java/org/archive/util/CachedBdbMapTest.java create mode 100644 commons/src/test/java/org/archive/util/FileUtilsTest.java create mode 100644 commons/src/test/java/org/archive/util/IoUtilsTest.java create mode 100644 commons/src/test/java/org/archive/util/MimetypeUtilsTest.java create mode 100644 commons/src/test/java/org/archive/util/PaddingStringBufferTest.java create mode 100644 commons/src/test/java/org/archive/util/SURTTest.java create mode 100644 commons/src/test/java/org/archive/util/SurtPrefixSetTest.java create mode 100644 commons/src/test/java/org/archive/util/TextUtilsTest.java create mode 100644 commons/src/test/java/org/archive/util/TransformTest.java create mode 100644 commons/src/test/java/org/archive/util/anvl/ANVLRecordTest.java create mode 100644 commons/src/test/java/org/archive/util/fingerprint/ArrayLongFPCacheTest.java create mode 100644 commons/src/test/java/org/archive/util/fingerprint/LongFPSetCacheTest.java create mode 100644 commons/src/test/java/org/archive/util/fingerprint/MemLongFPSetTest.java create mode 100644 commons/src/test/java/org/archive/util/ms/15336-doc-preface.doc create mode 100644 commons/src/test/java/org/archive/util/ms/15336-doc-preface.txt create mode 100644 commons/src/test/java/org/archive/util/ms/DocTest.java create mode 100644 commons/src/test/java/org/archive/util/ms/PieceReaderTest.java create mode 100644 commons/src/test/java/org/archive/util/ms/X.doc create mode 100644 commons/src/test/java/org/archive/util/ms/X.txt create mode 100644 commons/src/test/resources/org/archive/settings/path/anonymous.resolved.txt create mode 100644 commons/src/test/resources/org/archive/settings/path/global.get.txt create mode 100644 commons/src/test/resources/org/archive/settings/path/global.resolved.txt create mode 100644 commons/src/test/resources/org/archive/settings/path/o1.get.txt create mode 100644 commons/src/test/resources/org/archive/settings/path/o1.resolved.txt create mode 100644 dist/HOWTO-Launch-Heritrix.txt create mode 100644 dist/LICENSE.txt create mode 100644 dist/README.txt create mode 100644 dist/pom.xml create mode 100644 dist/src/main/assembly/dist.xml create mode 100644 dist/src/main/assembly/src.xml create mode 100755 dist/src/main/bin/arcreader create mode 100644 dist/src/main/bin/arcreader.cmd create mode 100644 dist/src/main/bin/dependencies.xsl create mode 100755 dist/src/main/bin/extractor create mode 100644 dist/src/main/bin/extractor.cmd create mode 100755 dist/src/main/bin/foreground_heritrix create mode 100644 dist/src/main/bin/foreground_heritrix.cmd create mode 100644 dist/src/main/bin/heritrix create mode 100644 dist/src/main/bin/heritrix.cmd create mode 100644 dist/src/main/bin/hoppath.pl create mode 100755 dist/src/main/bin/htmlextractor create mode 100644 dist/src/main/bin/htmlextractor.cmd create mode 100644 dist/src/main/bin/make_reports.pl create mode 100644 dist/src/main/bin/manifest_bundle.pl create mode 100644 dist/src/main/bin/xdocToTxt.xsl create mode 100644 dist/src/main/conf/heritrix.cacerts create mode 100644 dist/src/main/conf/jndi.properties create mode 100644 dist/src/main/conf/jobs/profile-defaults/defaults.xml create mode 100644 dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml create mode 100644 dist/src/main/conf/logging.properties create mode 100644 dist/src/main/extras/pagerank/GenGraph.java create mode 100644 dist/src/main/extras/pagerank/PageRank.java create mode 100644 dist/src/main/extras/pagerank/README.txt create mode 100755 dist/src/main/extras/pagerank/assignUrlIndex.pl create mode 100755 dist/src/main/extras/pagerank/pageRankSetup.pl create mode 100755 dist/src/main/extras/pagerank/run-pr.sh create mode 100644 dist/src/main/licenses/ant.LICENSE create mode 100644 dist/src/main/licenses/ant.NOTICE create mode 100644 dist/src/main/licenses/bsh.LICENSE create mode 100644 dist/src/main/licenses/commons-cli.LICENSE create mode 100644 dist/src/main/licenses/commons-cli.NOTICE create mode 100644 dist/src/main/licenses/commons-codec.LICENSE create mode 100644 dist/src/main/licenses/commons-codec.NOTICE create mode 100644 dist/src/main/licenses/commons-el.LICENSE create mode 100644 dist/src/main/licenses/commons-el.NOTICE create mode 100644 dist/src/main/licenses/commons-io.LICENSE create mode 100644 dist/src/main/licenses/commons-io.NOTICE create mode 100644 dist/src/main/licenses/commons-lang.LICENSE create mode 100644 dist/src/main/licenses/commons-lang.NOTICE create mode 100644 dist/src/main/licenses/commons-logging.LICENSE create mode 100644 dist/src/main/licenses/commons-net.LICENSE create mode 100644 dist/src/main/licenses/commons-net.NOTICE create mode 100644 dist/src/main/licenses/dnsjava-2.0.3.README create mode 100644 dist/src/main/licenses/fastutil-5.0.7.LICENSE create mode 100644 dist/src/main/licenses/itext.LICENSE create mode 100644 dist/src/main/licenses/jasper.LICENSE create mode 100644 dist/src/main/licenses/javaswf.LICENSE create mode 100644 dist/src/main/licenses/je-3.2.44.LICENSE create mode 100644 dist/src/main/licenses/jericho-html-2.3.LICENSE create mode 100644 dist/src/main/licenses/jets3t-0.5.0.LICENSE create mode 100644 dist/src/main/licenses/jetty.LICENSE create mode 100644 dist/src/main/licenses/junit.LICENSE.HTML create mode 100644 dist/src/main/licenses/libidn.LICENSE create mode 100644 dist/src/main/licenses/mg4j-1.0.1.LICENSE create mode 100644 dist/src/main/licenses/oro-2.0.8.LICENSE create mode 100644 dist/src/main/licenses/poi.LICENSE create mode 100644 dist/src/main/licenses/poi.NOTICE create mode 100644 dist/src/main/licenses/servlet-4.1.34.LICENSE create mode 100644 dist/src/test/java/org/archive/crawler/BasicProfileTest.java create mode 100644 engine/.classpath create mode 100644 engine/.cvsignore create mode 100644 engine/.project create mode 100644 engine/pom.xml create mode 100644 engine/src/design/credentials.gif create mode 100644 engine/src/design/credentials.zargo create mode 100644 engine/src/main/assembly/dist.xml create mode 100644 engine/src/main/java/META-INF/MANIFEST-MF create mode 100644 engine/src/main/java/org/archive/crawler/Heritrix.java create mode 100644 engine/src/main/java/org/archive/crawler/datamodel/CoreAttributeConstants.java create mode 100644 engine/src/main/java/org/archive/crawler/datamodel/CrawlURI.java create mode 100644 engine/src/main/java/org/archive/crawler/datamodel/SchedulingConstants.java create mode 100644 engine/src/main/java/org/archive/crawler/datamodel/UriUniqFilter.java create mode 100644 engine/src/main/java/org/archive/crawler/deciderules/ClassKeyMatchesRegExpDecideRule.java create mode 100644 engine/src/main/java/org/archive/crawler/deciderules/package.html create mode 100644 engine/src/main/java/org/archive/crawler/doc-files/processing_steps.dia create mode 100644 engine/src/main/java/org/archive/crawler/doc-files/processing_steps.png create mode 100644 engine/src/main/java/org/archive/crawler/event/CrawlStateEvent.java create mode 100644 engine/src/main/java/org/archive/crawler/event/CrawlURIDispositionEvent.java create mode 100644 engine/src/main/java/org/archive/crawler/event/StatSnapshotEvent.java create mode 100644 engine/src/main/java/org/archive/crawler/framework/Checkpointer.java create mode 100644 engine/src/main/java/org/archive/crawler/framework/CrawlController.java create mode 100644 engine/src/main/java/org/archive/crawler/framework/CrawlJob.java create mode 100644 engine/src/main/java/org/archive/crawler/framework/CrawlLimitEnforcer.java create mode 100644 engine/src/main/java/org/archive/crawler/framework/CrawlStatus.java create mode 100644 engine/src/main/java/org/archive/crawler/framework/Engine.java create mode 100644 engine/src/main/java/org/archive/crawler/framework/Frontier.java create mode 100644 engine/src/main/java/org/archive/crawler/framework/Scoper.java create mode 100644 engine/src/main/java/org/archive/crawler/framework/ToePool.java create mode 100644 engine/src/main/java/org/archive/crawler/framework/ToeThread.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/AntiCalendarCostAssignmentPolicy.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/BdbWorkQueue.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/BucketQueueAssignmentPolicy.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/CostAssignmentPolicy.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/FrontierJournal.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/HostnameQueueAssignmentPolicy.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/IPQueueAssignmentPolicy.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/QueueAssignmentPolicy.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/RecyclingSerialBinding.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/SurtAuthorityQueueAssignmentPolicy.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/TopmostAssignedSurtQueueAssignmentPolicy.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/UnitCostAssignmentPolicy.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/WagCostAssignmentPolicy.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/ZeroCostAssignmentPolicy.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/precedence/BaseQueuePrecedencePolicy.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/precedence/BaseUriPrecedencePolicy.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/precedence/CostUriPrecedencePolicy.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/precedence/HighestUriQueuePrecedencePolicy.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/precedence/HopsUriPrecedencePolicy.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceLoader.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceProvider.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/precedence/PreloadedUriPrecedencePolicy.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/precedence/QueuePrecedencePolicy.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/precedence/SimplePrecedenceProvider.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/precedence/SuccessCountsQueuePrecedencePolicy.java create mode 100644 engine/src/main/java/org/archive/crawler/frontier/precedence/UriPrecedencePolicy.java create mode 100644 engine/src/main/java/org/archive/crawler/io/NonFatalErrorFormatter.java create mode 100644 engine/src/main/java/org/archive/crawler/io/RuntimeErrorFormatter.java create mode 100644 engine/src/main/java/org/archive/crawler/io/StatisticsLogFormatter.java create mode 100644 engine/src/main/java/org/archive/crawler/io/UriErrorFormatter.java create mode 100644 engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java create mode 100644 engine/src/main/java/org/archive/crawler/package.html create mode 100644 engine/src/main/java/org/archive/crawler/postprocessor/CrawlStateUpdater.java create mode 100644 engine/src/main/java/org/archive/crawler/postprocessor/FrontierScheduler.java create mode 100644 engine/src/main/java/org/archive/crawler/postprocessor/LinksScoper.java create mode 100644 engine/src/main/java/org/archive/crawler/postprocessor/LowDiskPauseProcessor.java create mode 100644 engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java create mode 100644 engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java create mode 100644 engine/src/main/java/org/archive/crawler/prefetch/Preselector.java create mode 100644 engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java create mode 100644 engine/src/main/java/org/archive/crawler/prefetch/RuntimeLimitEnforcer.java create mode 100644 engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java create mode 100644 engine/src/main/java/org/archive/crawler/processor/HashCrawlMapper.java create mode 100644 engine/src/main/java/org/archive/crawler/processor/LexicalCrawlMapper.java create mode 100644 engine/src/main/java/org/archive/crawler/reporting/AlertHandler.java create mode 100644 engine/src/main/java/org/archive/crawler/reporting/AlertThreadGroup.java create mode 100644 engine/src/main/java/org/archive/crawler/reporting/CrawlStatSnapshot.java create mode 100644 engine/src/main/java/org/archive/crawler/reporting/CrawlSummaryReport.java create mode 100644 engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java create mode 100644 engine/src/main/java/org/archive/crawler/reporting/FrontierReport.java create mode 100644 engine/src/main/java/org/archive/crawler/reporting/HostsReport.java create mode 100644 engine/src/main/java/org/archive/crawler/reporting/MimetypesReport.java create mode 100644 engine/src/main/java/org/archive/crawler/reporting/ProcessorsReport.java create mode 100644 engine/src/main/java/org/archive/crawler/reporting/Report.java create mode 100644 engine/src/main/java/org/archive/crawler/reporting/ResponseCodeReport.java create mode 100644 engine/src/main/java/org/archive/crawler/reporting/SeedRecord.java create mode 100644 engine/src/main/java/org/archive/crawler/reporting/SeedsReport.java create mode 100644 engine/src/main/java/org/archive/crawler/reporting/SourceTagsReport.java create mode 100644 engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java create mode 100644 engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java create mode 100644 engine/src/main/java/org/archive/crawler/restlet/EngineApplication.java create mode 100644 engine/src/main/java/org/archive/crawler/restlet/EngineResource.java create mode 100644 engine/src/main/java/org/archive/crawler/restlet/EnhDirectory.java create mode 100644 engine/src/main/java/org/archive/crawler/restlet/EnhDirectoryResource.java create mode 100644 engine/src/main/java/org/archive/crawler/restlet/Flash.java create mode 100644 engine/src/main/java/org/archive/crawler/restlet/JobResource.java create mode 100644 engine/src/main/java/org/archive/crawler/restlet/PagedRepresentation.java create mode 100644 engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java create mode 100644 engine/src/main/java/org/archive/crawler/spring/SheetForDecideRuled.java create mode 100644 engine/src/main/java/org/archive/crawler/spring/SheetForSurtPrefixes.java create mode 100644 engine/src/main/java/org/archive/crawler/spring/SheetOverlaysManager.java create mode 100644 engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java create mode 100644 engine/src/main/java/org/archive/crawler/util/BenchmarkUriUniqFilters.java create mode 100644 engine/src/main/java/org/archive/crawler/util/BloomUriUniqFilter.java create mode 100644 engine/src/main/java/org/archive/crawler/util/CheckpointUtils.java create mode 100644 engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java create mode 100644 engine/src/main/java/org/archive/crawler/util/DiskFPMergeUriUniqFilter.java create mode 100644 engine/src/main/java/org/archive/crawler/util/FPMergeUriUniqFilter.java create mode 100644 engine/src/main/java/org/archive/crawler/util/FPUriUniqFilter.java create mode 100644 engine/src/main/java/org/archive/crawler/util/LogReader.java create mode 100644 engine/src/main/java/org/archive/crawler/util/LogUtils.java create mode 100644 engine/src/main/java/org/archive/crawler/util/Logs.java create mode 100644 engine/src/main/java/org/archive/crawler/util/MemFPMergeUriUniqFilter.java create mode 100644 engine/src/main/java/org/archive/crawler/util/MemUriUniqFilter.java create mode 100644 engine/src/main/java/org/archive/crawler/util/NoopUriUniqFilter.java create mode 100644 engine/src/main/java/org/archive/crawler/util/RecoveryLogMapper.java create mode 100644 engine/src/main/java/org/archive/crawler/util/SeedUrlNotFoundException.java create mode 100644 engine/src/main/java/org/archive/crawler/util/SetBasedUriUniqFilter.java create mode 100644 engine/src/main/java/org/archive/crawler/util/Sorts.java create mode 100644 engine/src/main/java/org/archive/crawler/util/StringIntPair.java create mode 100644 engine/src/main/java/org/archive/crawler/util/StringIntPairComparator.java create mode 100644 engine/src/main/java/org/archive/crawler/util/TopNSet.java create mode 100644 engine/src/main/java/org/archive/extractor/CharSequenceLinkExtractor.java create mode 100644 engine/src/main/java/org/archive/extractor/CharSequenceProvider.java create mode 100644 engine/src/main/java/org/archive/extractor/ExtractErrorListener.java create mode 100644 engine/src/main/java/org/archive/extractor/LinkExtractor.java create mode 100644 engine/src/main/java/org/archive/extractor/RegexpCSSLinkExtractor.java create mode 100644 engine/src/main/java/org/archive/extractor/RegexpHTMLLinkExtractor.java create mode 100644 engine/src/main/java/org/archive/extractor/RegexpJSLinkExtractor.java create mode 100644 engine/src/main/java/org/archive/extractor/overview.html create mode 100644 engine/src/main/java/org/archive/overview.html create mode 100644 engine/src/main/resources/arcMetaheaderBody.xsl create mode 100644 engine/src/main/resources/org/archive/crawler/extras/adaptive/AdaptiveRevisitFrontier_en.utf8 create mode 100644 engine/src/main/resources/org/archive/crawler/extras/adaptive/WaitEvaluator_en.utf8 create mode 100644 engine/src/main/resources/org/archive/crawler/framework/CrawlControllerImpl_en.utf8 create mode 100644 engine/src/main/resources/org/archive/crawler/framework/CrawlScope_en.utf8 create mode 100644 engine/src/main/resources/org/archive/crawler/framework/CrawlerProcessor_en.utf8 create mode 100644 engine/src/main/resources/org/archive/crawler/framework/Scoper_en.utf8 create mode 100644 engine/src/main/resources/org/archive/crawler/frontier/AbstractFrontier_en.utf8 create mode 100644 engine/src/main/resources/org/archive/crawler/frontier/BdbFrontier_en.utf8 create mode 100644 engine/src/main/resources/org/archive/crawler/frontier/BucketQueueAssignmentPolicy_en.utf8 create mode 100644 engine/src/main/resources/org/archive/crawler/frontier/WorkQueueFrontier_en.utf8 create mode 100644 engine/src/main/resources/org/archive/crawler/frontier/precedence/BaseQueuePrecedencePolicy_en.utf8 create mode 100644 engine/src/main/resources/org/archive/crawler/frontier/precedence/BaseUriPrecedencePolicy_en.utf8 create mode 100644 engine/src/main/resources/org/archive/crawler/frontier/precedence/CostUriPrecedencePolicy_en.utf8 create mode 100644 engine/src/main/resources/org/archive/crawler/frontier/precedence/HighestUriQueuePrecedencePolicy_en.utf8 create mode 100644 engine/src/main/resources/org/archive/crawler/frontier/precedence/HopsUriPrecedencePolicy_en.utf8 create mode 100644 engine/src/main/resources/org/archive/crawler/frontier/precedence/SuccessCountsQueuePrecedencePolicy_en.utf8 create mode 100644 engine/src/main/resources/org/archive/crawler/postprocessor/CrawlStateUpdater_en.utf8 create mode 100644 engine/src/main/resources/org/archive/crawler/postprocessor/FrontierScheduler_en.utf8 create mode 100644 engine/src/main/resources/org/archive/crawler/postprocessor/LinksScoper_en.utf8 create mode 100644 engine/src/main/resources/org/archive/crawler/postprocessor/LowDiskPauseProcessor_en.utf8 create mode 100644 engine/src/main/resources/org/archive/crawler/postprocessor/SupplementaryLinksScoper_en.utf8 create mode 100644 engine/src/main/resources/org/archive/crawler/prefetch/PreconditionEnforcer_en.utf8 create mode 100644 engine/src/main/resources/org/archive/crawler/prefetch/Preselector_en.utf8 create mode 100644 engine/src/main/resources/org/archive/crawler/prefetch/QuotaEnforcer_en.utf8 create mode 100644 engine/src/main/resources/org/archive/crawler/prefetch/RuntimeLimitEnforcer_en.utf8 create mode 100644 engine/src/main/resources/org/archive/crawler/processor/CrawlMapper_en.utf8 create mode 100644 engine/src/main/resources/org/archive/crawler/processor/HashCrawlMapper_en.utf8 create mode 100644 engine/src/main/resources/org/archive/crawler/processor/LexicalCrawlMapper_en.utf8 create mode 100644 engine/src/test/java/org/archive/crawler/datamodel/CandidateURITest.java create mode 100644 engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java create mode 100644 engine/src/test/java/org/archive/crawler/framework/CrawlControllerTest.java create mode 100644 engine/src/test/java/org/archive/crawler/framework/CrawlerProcessorTestBase.java create mode 100644 engine/src/test/java/org/archive/crawler/frontier/BdbFrontierTest.java create mode 100644 engine/src/test/java/org/archive/crawler/frontier/BdbMultipleWorkQueuesTest.java create mode 100644 engine/src/test/java/org/archive/crawler/frontier/BucketQueueAssignmentPolicyTest.java create mode 100644 engine/src/test/java/org/archive/crawler/frontier/FrontierJournalTest.java create mode 100644 engine/src/test/java/org/archive/crawler/frontier/precedence/BaseQueuePrecedencePolicyTest.java create mode 100644 engine/src/test/java/org/archive/crawler/frontier/precedence/BaseUriPrecedencePolicyTest.java create mode 100644 engine/src/test/java/org/archive/crawler/frontier/precedence/CostUriPrecedencePolicyTest.java create mode 100644 engine/src/test/java/org/archive/crawler/frontier/precedence/HighestUriQueuePrecedencePolicyTest.java create mode 100644 engine/src/test/java/org/archive/crawler/frontier/precedence/HopsUriPrecedencePolicyTest.java create mode 100644 engine/src/test/java/org/archive/crawler/frontier/precedence/SuccessCountsQueuePrecedencePolicyTest.java create mode 100644 engine/src/test/java/org/archive/crawler/postprocessor/CrawlStateUpdaterTest.java create mode 100644 engine/src/test/java/org/archive/crawler/postprocessor/FrontierSchedulerTest.java create mode 100644 engine/src/test/java/org/archive/crawler/postprocessor/LinksScoperTest.java create mode 100644 engine/src/test/java/org/archive/crawler/postprocessor/LowDiskPauseProcessorTest.java create mode 100644 engine/src/test/java/org/archive/crawler/postprocessor/SupplementaryLinksScoperTest.java create mode 100644 engine/src/test/java/org/archive/crawler/prefetch/PreconditionEnforcerTest.java create mode 100644 engine/src/test/java/org/archive/crawler/prefetch/PreselectorTest.java create mode 100644 engine/src/test/java/org/archive/crawler/prefetch/QuotaEnforcerTest.java create mode 100644 engine/src/test/java/org/archive/crawler/prefetch/RuntimeLimitEnforcerTest.java create mode 100644 engine/src/test/java/org/archive/crawler/processor/HashCrawlMapperTest.java create mode 100644 engine/src/test/java/org/archive/crawler/processor/LexicalCrawlMapperTest.java create mode 100644 engine/src/test/java/org/archive/crawler/selftest/BackgroundImageExtractionSelfTestCase.java create mode 100644 engine/src/test/java/org/archive/crawler/selftest/BadURIsStopPageParsingSelfTest.java create mode 100644 engine/src/test/java/org/archive/crawler/selftest/CharsetSelfTest.java create mode 100644 engine/src/test/java/org/archive/crawler/selftest/CheckpointSelfTest.java create mode 100644 engine/src/test/java/org/archive/crawler/selftest/FormAuthSelfTest.java create mode 100644 engine/src/test/java/org/archive/crawler/selftest/FormAuthServlet.java create mode 100644 engine/src/test/java/org/archive/crawler/selftest/FramesSelfTestCase.java create mode 100644 engine/src/test/java/org/archive/crawler/selftest/HttpAuthSelfTest.java create mode 100644 engine/src/test/java/org/archive/crawler/selftest/KeyWordProcessor.java create mode 100644 engine/src/test/java/org/archive/crawler/selftest/KeyWordUriPrecedencePolicy.java create mode 100644 engine/src/test/java/org/archive/crawler/selftest/MaxLinkHopsSelfTest.java create mode 100644 engine/src/test/java/org/archive/crawler/selftest/Precedence1SelfTest.java create mode 100644 engine/src/test/java/org/archive/crawler/selftest/Precedence2SelfTest.java create mode 100644 engine/src/test/java/org/archive/crawler/selftest/Precedence3SelfTest.java create mode 100644 engine/src/test/java/org/archive/crawler/selftest/Precedence4SelfTest.java create mode 100644 engine/src/test/java/org/archive/crawler/selftest/RandomServlet.java create mode 100644 engine/src/test/java/org/archive/crawler/selftest/RandomServletTest.java create mode 100644 engine/src/test/java/org/archive/crawler/selftest/SelfTestBase.java create mode 100644 engine/src/test/java/org/archive/crawler/selftest/SimpleSelfTest.java create mode 100644 engine/src/test/java/org/archive/crawler/selftest/UserAgentSelfTest.java create mode 100644 engine/src/test/java/org/archive/crawler/selftest/UserAgentServlet.java create mode 100644 engine/src/test/java/org/archive/crawler/selftest/package.html create mode 100644 engine/src/test/java/org/archive/crawler/util/BdbUriUniqFilterTest.java create mode 100644 engine/src/test/java/org/archive/crawler/util/BloomUriUniqFilterTest.java create mode 100644 engine/src/test/java/org/archive/crawler/util/FPUriUniqFilterTest.java create mode 100644 engine/src/webapps/selftest/Auth/basic/basic-loggedin.html create mode 100644 engine/src/webapps/selftest/Auth/basic/index.html create mode 100644 engine/src/webapps/selftest/Auth/form/get/error.html create mode 100644 engine/src/webapps/selftest/Auth/form/get/get-loggedin.html create mode 100644 engine/src/webapps/selftest/Auth/form/get/index.html create mode 100644 engine/src/webapps/selftest/Auth/form/get/success.jsp create mode 100644 engine/src/webapps/selftest/Auth/form/index.html create mode 100644 engine/src/webapps/selftest/Auth/form/post/error.html create mode 100644 engine/src/webapps/selftest/Auth/form/post/index.html create mode 100644 engine/src/webapps/selftest/Auth/form/post/post-loggedin.html create mode 100644 engine/src/webapps/selftest/Auth/form/post/success.jsp create mode 100644 engine/src/webapps/selftest/Auth/index.html create mode 100644 engine/src/webapps/selftest/BackgroundImageExtraction/example-background-image.jpeg create mode 100644 engine/src/webapps/selftest/BackgroundImageExtraction/index.html create mode 100644 engine/src/webapps/selftest/BadURIsStopPageParsing/goodone.html create mode 100644 engine/src/webapps/selftest/BadURIsStopPageParsing/goodthree.html create mode 100644 engine/src/webapps/selftest/BadURIsStopPageParsing/goodtwo.html create mode 100644 engine/src/webapps/selftest/BadURIsStopPageParsing/index.html create mode 100644 engine/src/webapps/selftest/BadURIsStopPageParsing/one.html create mode 100644 engine/src/webapps/selftest/BadURIsStopPageParsing/three.html create mode 100644 engine/src/webapps/selftest/BadURIsStopPageParsing/two.html create mode 100644 engine/src/webapps/selftest/Charset/charsetselftest_end.html create mode 100644 engine/src/webapps/selftest/Charset/index.html create mode 100644 engine/src/webapps/selftest/Charset/shiftjis.jsp create mode 100644 engine/src/webapps/selftest/Charset/utf8.jsp create mode 100644 engine/src/webapps/selftest/Checkpoint/index.html create mode 100644 engine/src/webapps/selftest/FlashParse/index.html create mode 100644 engine/src/webapps/selftest/FlashParse/pirates.swf create mode 100644 engine/src/webapps/selftest/FlashParse/success.html create mode 100644 engine/src/webapps/selftest/FormTagExtraction/index.html create mode 100644 engine/src/webapps/selftest/FormTagExtraction/inputtag.html create mode 100644 engine/src/webapps/selftest/FormTagExtraction/optiontag.html create mode 100644 engine/src/webapps/selftest/Frames/index.html create mode 100644 engine/src/webapps/selftest/Frames/leftframe.html create mode 100644 engine/src/webapps/selftest/Frames/noframe.html create mode 100644 engine/src/webapps/selftest/Frames/rightframe.html create mode 100644 engine/src/webapps/selftest/Frames/topframe.html create mode 100644 engine/src/webapps/selftest/MaxLinkHops/1.html create mode 100644 engine/src/webapps/selftest/MaxLinkHops/2.html create mode 100644 engine/src/webapps/selftest/MaxLinkHops/3.html create mode 100644 engine/src/webapps/selftest/MaxLinkHops/4.html create mode 100644 engine/src/webapps/selftest/MaxLinkHops/5.html create mode 100644 engine/src/webapps/selftest/MaxLinkHops/6.html create mode 100644 engine/src/webapps/selftest/MaxLinkHops/index.html create mode 100644 engine/src/webapps/selftest/Refresh/index.html create mode 100644 engine/src/webapps/selftest/Refresh/refresh.html create mode 100644 engine/src/webapps/selftest/Refresh/refresh2.html create mode 100644 engine/src/webapps/selftest/RobotsExclusion/README.txt create mode 100644 engine/src/webapps/selftest/RobotsExclusion/excluded.html create mode 100644 engine/src/webapps/selftest/RobotsExclusion/excluded/level2/level3/excluded.html create mode 100644 engine/src/webapps/selftest/RobotsExclusion/included.html create mode 100644 engine/src/webapps/selftest/RobotsExclusion/index.html create mode 100644 engine/src/webapps/selftest/SimpleDocumentTypes/binaries/avi.avi create mode 100644 engine/src/webapps/selftest/SimpleDocumentTypes/binaries/doc.doc create mode 100644 engine/src/webapps/selftest/SimpleDocumentTypes/binaries/jpg.jpg create mode 100644 engine/src/webapps/selftest/SimpleDocumentTypes/binaries/mp3.mp3 create mode 100644 engine/src/webapps/selftest/SimpleDocumentTypes/binaries/pdf.pdf create mode 100644 engine/src/webapps/selftest/SimpleDocumentTypes/binaries/ppt.ppt create mode 100644 engine/src/webapps/selftest/SimpleDocumentTypes/binaries/ps.ps create mode 100644 engine/src/webapps/selftest/SimpleDocumentTypes/binaries/rtf.rtf create mode 100644 engine/src/webapps/selftest/SimpleDocumentTypes/binaries/wav.wav create mode 100644 engine/src/webapps/selftest/SimpleDocumentTypes/binaries/wpd.wpd create mode 100644 engine/src/webapps/selftest/SimpleDocumentTypes/binaries/xls.xls create mode 100644 engine/src/webapps/selftest/SimpleDocumentTypes/html.html create mode 100644 engine/src/webapps/selftest/SimpleDocumentTypes/index.html create mode 100644 engine/src/webapps/selftest/SimpleDocumentTypes/txt.txt create mode 100644 engine/src/webapps/selftest/SimpleJavascriptExtraction/index.html create mode 100644 engine/src/webapps/selftest/SimpleJavascriptExtraction/jscriptOpenWindow.html create mode 100644 engine/src/webapps/selftest/SimpleJavascriptExtraction/jscriptOpenWindowArg.html create mode 100644 engine/src/webapps/selftest/SpacesInHrefPath/index.html create mode 100644 engine/src/webapps/selftest/SpacesInHrefPath/spaces in path.html create mode 100644 engine/src/webapps/selftest/TrickyRelativeURIs/anothersub/reluptarget.html create mode 100644 engine/src/webapps/selftest/TrickyRelativeURIs/index.html create mode 100644 engine/src/webapps/selftest/TrickyRelativeURIs/reluptricky.html create mode 100644 engine/src/webapps/selftest/WEB-INF/web.xml create mode 100644 engine/src/webapps/selftest/index.jsp create mode 100644 engine/src/webapps/selftest/robots.txt create mode 100644 engine/src/xsd/arc/1.0/arc.html create mode 100644 engine/src/xsd/arc/1.0/arc.xsd create mode 100644 engine/src/xsd/arc/1.0/example.xml create mode 100644 engine/testdata/selftest/BackgroundImageExtractionSelfTestCase/htdocs/example-background-image.jpeg create mode 100644 engine/testdata/selftest/BackgroundImageExtractionSelfTestCase/htdocs/index.html create mode 100644 engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/goodone.html create mode 100644 engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/goodthree.html create mode 100644 engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/goodtwo.html create mode 100644 engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/index.html create mode 100644 engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/one.html create mode 100644 engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/three.html create mode 100644 engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/two.html create mode 100644 engine/testdata/selftest/CharsetSelfTest/htdocs/index.html create mode 100644 engine/testdata/selftest/CharsetSelfTest/htdocs/link.html create mode 100644 engine/testdata/selftest/CheckpointSelfTest/profile/config.txt create mode 100644 engine/testdata/selftest/CheckpointSelfTest/profile/seeds.txt create mode 100644 engine/testdata/selftest/CheckpointSelfTest/profile/sheets/default.single create mode 100644 engine/testdata/selftest/FlashParseSelfTest/htdocs/index.html create mode 100644 engine/testdata/selftest/FlashParseSelfTest/htdocs/pirates.swf create mode 100644 engine/testdata/selftest/FlashParseSelfTest/htdocs/success.html create mode 100644 engine/testdata/selftest/FormAuthSelfTest/htdocs/failure.html create mode 100644 engine/testdata/selftest/FormAuthSelfTest/htdocs/index.html create mode 100644 engine/testdata/selftest/FormAuthSelfTest/htdocs/link1.html create mode 100644 engine/testdata/selftest/FormAuthSelfTest/htdocs/link2.html create mode 100644 engine/testdata/selftest/FormAuthSelfTest/htdocs/link3.html create mode 100644 engine/testdata/selftest/FormAuthSelfTest/htdocs/success.html create mode 100644 engine/testdata/selftest/FramesSelfTestCase/htdocs/index.html create mode 100644 engine/testdata/selftest/FramesSelfTestCase/htdocs/leftframe.html create mode 100644 engine/testdata/selftest/FramesSelfTestCase/htdocs/noframe.html create mode 100644 engine/testdata/selftest/FramesSelfTestCase/htdocs/rightframe.html create mode 100644 engine/testdata/selftest/FramesSelfTestCase/htdocs/topframe.html create mode 100644 engine/testdata/selftest/HttpAuthSelfTest/htdocs/basic/index.html create mode 100644 engine/testdata/selftest/HttpAuthSelfTest/htdocs/basic/link1.html create mode 100644 engine/testdata/selftest/HttpAuthSelfTest/htdocs/basic/link2.html create mode 100644 engine/testdata/selftest/HttpAuthSelfTest/htdocs/basic/link3.html create mode 100644 engine/testdata/selftest/HttpAuthSelfTest/htdocs/failure.html create mode 100644 engine/testdata/selftest/HttpAuthSelfTest/htdocs/index.html create mode 100644 engine/testdata/selftest/HttpAuthSelfTest/htdocs/link1.html create mode 100644 engine/testdata/selftest/HttpAuthSelfTest/htdocs/link2.html create mode 100644 engine/testdata/selftest/HttpAuthSelfTest/htdocs/link3.html create mode 100644 engine/testdata/selftest/HttpAuthSelfTest/htdocs/success.html create mode 100644 engine/testdata/selftest/MaxLinkHopsSelfTest/htdocs/1.html create mode 100644 engine/testdata/selftest/MaxLinkHopsSelfTest/htdocs/2.html create mode 100644 engine/testdata/selftest/MaxLinkHopsSelfTest/htdocs/3.html create mode 100644 engine/testdata/selftest/MaxLinkHopsSelfTest/htdocs/4.html create mode 100644 engine/testdata/selftest/MaxLinkHopsSelfTest/htdocs/5.html create mode 100644 engine/testdata/selftest/MaxLinkHopsSelfTest/htdocs/index.html create mode 100644 engine/testdata/selftest/Precedence1SelfTest/htdocs/five/a.html create mode 100644 engine/testdata/selftest/Precedence1SelfTest/htdocs/five/b.html create mode 100644 engine/testdata/selftest/Precedence1SelfTest/htdocs/one/a.html create mode 100644 engine/testdata/selftest/Precedence1SelfTest/htdocs/one/b.html create mode 100644 engine/testdata/selftest/Precedence1SelfTest/htdocs/seed.html create mode 100644 engine/testdata/selftest/Precedence1SelfTest/htdocs/ten/a.html create mode 100644 engine/testdata/selftest/Precedence1SelfTest/htdocs/ten/b.html create mode 100644 engine/testdata/selftest/Precedence2SelfTest/htdocs/five/a.html create mode 100644 engine/testdata/selftest/Precedence2SelfTest/htdocs/five/b.html create mode 100644 engine/testdata/selftest/Precedence2SelfTest/htdocs/one/a.html create mode 100644 engine/testdata/selftest/Precedence2SelfTest/htdocs/one/b.html create mode 100644 engine/testdata/selftest/Precedence2SelfTest/htdocs/seed.html create mode 100644 engine/testdata/selftest/Precedence2SelfTest/htdocs/ten/a.html create mode 100644 engine/testdata/selftest/Precedence2SelfTest/htdocs/ten/b.html create mode 100644 engine/testdata/selftest/Precedence2SelfTest/profile/rank.txt create mode 100644 engine/testdata/selftest/Precedence3SelfTest/htdocs/A.html create mode 100644 engine/testdata/selftest/Precedence3SelfTest/htdocs/B.html create mode 100644 engine/testdata/selftest/Precedence3SelfTest/htdocs/C.html create mode 100644 engine/testdata/selftest/Precedence3SelfTest/htdocs/D.html create mode 100644 engine/testdata/selftest/Precedence3SelfTest/htdocs/E.html create mode 100644 engine/testdata/selftest/Precedence3SelfTest/htdocs/F.html create mode 100644 engine/testdata/selftest/Precedence3SelfTest/htdocs/G.html create mode 100644 engine/testdata/selftest/Precedence3SelfTest/htdocs/H.html create mode 100644 engine/testdata/selftest/Precedence3SelfTest/htdocs/I.html create mode 100644 engine/testdata/selftest/Precedence3SelfTest/htdocs/J.html create mode 100644 engine/testdata/selftest/Precedence3SelfTest/htdocs/K.html create mode 100644 engine/testdata/selftest/Precedence3SelfTest/htdocs/L.html create mode 100644 engine/testdata/selftest/Precedence3SelfTest/htdocs/M.html create mode 100644 engine/testdata/selftest/Precedence3SelfTest/htdocs/N.html create mode 100644 engine/testdata/selftest/Precedence3SelfTest/htdocs/O.html create mode 100644 engine/testdata/selftest/Precedence4SelfTest/htdocs/five/a.html create mode 100644 engine/testdata/selftest/Precedence4SelfTest/htdocs/five/b.html create mode 100644 engine/testdata/selftest/Precedence4SelfTest/htdocs/one/a.html create mode 100644 engine/testdata/selftest/Precedence4SelfTest/htdocs/one/b.html create mode 100644 engine/testdata/selftest/Precedence4SelfTest/htdocs/seed.html create mode 100644 engine/testdata/selftest/Precedence4SelfTest/htdocs/ten/a.html create mode 100644 engine/testdata/selftest/Precedence4SelfTest/htdocs/ten/b.html create mode 100644 engine/testdata/selftest/SimpleSelfTest/htdocs/index.html create mode 100644 engine/testdata/selftest/SimpleSelfTest/htdocs/link1.html create mode 100644 engine/testdata/selftest/SimpleSelfTest/htdocs/link2.html create mode 100644 engine/testdata/selftest/SimpleSelfTest/htdocs/link3.html create mode 100644 engine/testdata/selftest/conf/heritrix.properties create mode 100644 engine/testdata/selftest/conf/selftest-crawler-beans.cxml create mode 100644 modules/.classpath create mode 100644 modules/.project create mode 100644 modules/.settings/org.eclipse.jdt.core.prefs create mode 100644 modules/pom.xml create mode 100644 modules/src/main/java/org/archive/modules/BeanShellProcessor.java create mode 100644 modules/src/main/java/org/archive/modules/CrawlMetadata.java create mode 100644 modules/src/main/java/org/archive/modules/DefaultProcessorURI.java create mode 100644 modules/src/main/java/org/archive/modules/ModuleAttributeConstants.java create mode 100644 modules/src/main/java/org/archive/modules/PostProcessor.java create mode 100644 modules/src/main/java/org/archive/modules/ProcessResult.java create mode 100644 modules/src/main/java/org/archive/modules/ProcessStatus.java create mode 100644 modules/src/main/java/org/archive/modules/Processor.java create mode 100644 modules/src/main/java/org/archive/modules/ProcessorChain.java create mode 100644 modules/src/main/java/org/archive/modules/ProcessorLevel.java create mode 100644 modules/src/main/java/org/archive/modules/ProcessorTestBase.java create mode 100644 modules/src/main/java/org/archive/modules/ProcessorURI.java create mode 100644 modules/src/main/java/org/archive/modules/canonicalize/BaseRule.java create mode 100644 modules/src/main/java/org/archive/modules/canonicalize/CanonicalizationRule.java create mode 100644 modules/src/main/java/org/archive/modules/canonicalize/FixupQueryString.java create mode 100644 modules/src/main/java/org/archive/modules/canonicalize/LowercaseRule.java create mode 100644 modules/src/main/java/org/archive/modules/canonicalize/RegexRule.java create mode 100644 modules/src/main/java/org/archive/modules/canonicalize/RulesCanonicalizationPolicy.java create mode 100644 modules/src/main/java/org/archive/modules/canonicalize/StripExtraSlashes.java create mode 100644 modules/src/main/java/org/archive/modules/canonicalize/StripSessionCFIDs.java create mode 100644 modules/src/main/java/org/archive/modules/canonicalize/StripSessionIDs.java create mode 100644 modules/src/main/java/org/archive/modules/canonicalize/StripUserinfoRule.java create mode 100644 modules/src/main/java/org/archive/modules/canonicalize/StripWWWNRule.java create mode 100644 modules/src/main/java/org/archive/modules/canonicalize/StripWWWRule.java create mode 100644 modules/src/main/java/org/archive/modules/canonicalize/UriCanonicalizationPolicy.java create mode 100644 modules/src/main/java/org/archive/modules/credential/Credential.java create mode 100644 modules/src/main/java/org/archive/modules/credential/CredentialAvatar.java create mode 100644 modules/src/main/java/org/archive/modules/credential/CredentialStore.java create mode 100644 modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java create mode 100644 modules/src/main/java/org/archive/modules/credential/Rfc2617Credential.java create mode 100644 modules/src/main/java/org/archive/modules/credential/package.html create mode 100644 modules/src/main/java/org/archive/modules/deciderules/AcceptDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/AddRedirectFromRootServerToScope.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/BeanShellDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/ContentLengthDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/ContentTypeMatchesRegExpDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/ContentTypeNotMatchesRegExpDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/DecideResult.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/DecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/DecideRuleSequence.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/ExceedsDocumentLengthThresholdDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/ExternalGeoLocationDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/ExternalGeoLookupInterface.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/FetchStatusDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/FetchStatusMatchesRegExpDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/FetchStatusNotMatchesRegExpDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/HasViaDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/HopsPathMatchesRegExpDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/MatchesFilePatternDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/MatchesListRegExpDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/MatchesRegExpDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/NotExceedsDocumentLengthTresholdDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/NotMatchesFilePatternDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/NotMatchesListRegExpDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/NotMatchesRegExpDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/PathologicalPathDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/PredicatedDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/PrerequisiteAcceptDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/RejectDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/SeedAcceptDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/TooManyHopsDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/TooManyPathSegmentsDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/TransclusionDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/recrawl/IdenticalDigestDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/surt/NotOnDomainsDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/surt/NotOnHostsDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/surt/NotSurtPrefixedDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/surt/OnDomainsDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/surt/OnHostsDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/surt/ScopePlusOneDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/AggressiveExtractorHTML.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/ContentExtractor.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/ContentExtractorTestBase.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/CustomSWFTags.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/Extractor.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/ExtractorImpliedURI.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/ExtractorParameters.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/HTMLLinkContext.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/HTTPContentDigest.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/Hop.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/Link.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/LinkContext.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/PDFParser.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/StringExtractorTestBase.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/TempDirProvider.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/TrapSuppressExtractor.java create mode 100644 modules/src/main/java/org/archive/modules/extractor/UriErrorLoggerModule.java create mode 100644 modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStorage.java create mode 100644 modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java create mode 100644 modules/src/main/java/org/archive/modules/fetcher/CookieStorage.java create mode 100644 modules/src/main/java/org/archive/modules/fetcher/DefaultServerCache.java create mode 100644 modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java create mode 100644 modules/src/main/java/org/archive/modules/fetcher/FetchErrors.java create mode 100644 modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java create mode 100644 modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java create mode 100644 modules/src/main/java/org/archive/modules/fetcher/FetchStats.java create mode 100644 modules/src/main/java/org/archive/modules/fetcher/FetchStatusCodes.java create mode 100644 modules/src/main/java/org/archive/modules/fetcher/HeritrixHttpMethodRetryHandler.java create mode 100644 modules/src/main/java/org/archive/modules/fetcher/HeritrixProtocolSocketFactory.java create mode 100644 modules/src/main/java/org/archive/modules/fetcher/HeritrixSSLProtocolSocketFactory.java create mode 100644 modules/src/main/java/org/archive/modules/fetcher/HostResolver.java create mode 100644 modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStorage.java create mode 100644 modules/src/main/java/org/archive/modules/fetcher/UserAgentProvider.java create mode 100644 modules/src/main/java/org/archive/modules/net/BdbServerCache.java create mode 100644 modules/src/main/java/org/archive/modules/net/CrawlHost.java create mode 100644 modules/src/main/java/org/archive/modules/net/CrawlServer.java create mode 100644 modules/src/main/java/org/archive/modules/net/DefaultTempDirProvider.java create mode 100644 modules/src/main/java/org/archive/modules/net/RobotsDirectives.java create mode 100644 modules/src/main/java/org/archive/modules/net/RobotsExclusionPolicy.java create mode 100644 modules/src/main/java/org/archive/modules/net/RobotsHonoringPolicy.java create mode 100644 modules/src/main/java/org/archive/modules/net/Robotstxt.java create mode 100644 modules/src/main/java/org/archive/modules/net/ServerCache.java create mode 100644 modules/src/main/java/org/archive/modules/net/ServerCacheUtil.java create mode 100644 modules/src/main/java/org/archive/modules/package-info.java create mode 100644 modules/src/main/java/org/archive/modules/recrawl/FetchHistoryProcessor.java create mode 100644 modules/src/main/java/org/archive/modules/recrawl/PersistLoadProcessor.java create mode 100644 modules/src/main/java/org/archive/modules/recrawl/PersistLogProcessor.java create mode 100644 modules/src/main/java/org/archive/modules/recrawl/PersistOnlineProcessor.java create mode 100644 modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java create mode 100644 modules/src/main/java/org/archive/modules/recrawl/PersistStoreProcessor.java create mode 100644 modules/src/main/java/org/archive/modules/recrawl/RecrawlAttributeConstants.java create mode 100644 modules/src/main/java/org/archive/modules/seeds/SeedFileIterator.java create mode 100644 modules/src/main/java/org/archive/modules/seeds/SeedListener.java create mode 100644 modules/src/main/java/org/archive/modules/seeds/SeedModule.java create mode 100644 modules/src/main/java/org/archive/modules/seeds/TextSeedModule.java create mode 100644 modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java create mode 100644 modules/src/main/java/org/archive/modules/writer/Kw3Constants.java create mode 100644 modules/src/main/java/org/archive/modules/writer/Kw3WriterProcessor.java create mode 100644 modules/src/main/java/org/archive/modules/writer/MirrorWriterProcessor.java create mode 100644 modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java create mode 100644 modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java create mode 100644 modules/src/main/resources/org/archive/modules/BeanShellProcessor_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/Processor_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/canonicalize/BaseRule_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/canonicalize/FixupQueryString_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/canonicalize/LowercaseRule_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/canonicalize/RegexRule_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/canonicalize/StripExtraSlashes_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/canonicalize/StripSessionCFIDs_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/canonicalize/StripSessionIDs_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/canonicalize/StripUserinfoRule_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/canonicalize/StripWWWNRule_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/canonicalize/StripWWWRule_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/credential/CredentialStore_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/credential/Credential_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/credential/HtmlFormCredential_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/credential/Rfc2617Credential_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/deciderules/DecideRuleSequence_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/deciderules/DecideRule_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/deciderules/HasViaDecideRule_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/deciderules/HopsPathMatchesRegExpDecideRule_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/deciderules/MatchesRegExpDecideRule_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/deciderules/PredicatedAcceptDecideRule_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/extractor/AggressiveExtractorHTML_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/extractor/ExtractorCSS_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/extractor/ExtractorDOC_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/extractor/ExtractorHTML_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/extractor/ExtractorHTTP_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/extractor/ExtractorImpliedURI_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/extractor/ExtractorJS_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/extractor/ExtractorPDF_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/extractor/ExtractorSWF_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/extractor/ExtractorURI_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/extractor/ExtractorUniversal_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/extractor/ExtractorXML_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/extractor/Extractor_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/extractor/HTTPContentDigest_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/extractor/TrapSuppressExtractor_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/fetcher/FetchDNS_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/fetcher/FetchFTP_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/fetcher/FetchHTTP_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/net/RobotsHonoringPolicy_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/recrawl/FetchHistoryProcessor_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/recrawl/PersistLoadProcessor_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/recrawl/PersistLogProcessor_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/recrawl/PersistOnlineProcessor_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/recrawl/PersistStoreProcessor_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/writer/ARCWriterProcessor_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/writer/MirrorWriterProcessor_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/writer/WARCWriterProcessor_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/writer/WriterPoolProcessor_en.utf8 create mode 100644 modules/src/main/resources/org/archive/modules/writer/arc_metadata_template.xml create mode 100644 modules/src/test/java/org/archive/modules/BeanShellProcessorTest.java create mode 100644 modules/src/test/java/org/archive/modules/canonicalize/FixupQueryStringTest.java create mode 100644 modules/src/test/java/org/archive/modules/canonicalize/LowercaseRuleTest.java create mode 100644 modules/src/test/java/org/archive/modules/canonicalize/RegexRuleTest.java create mode 100644 modules/src/test/java/org/archive/modules/canonicalize/RulesCanonicalizationPolicyTest.java create mode 100644 modules/src/test/java/org/archive/modules/canonicalize/StripSessionCFIDsTest.java create mode 100644 modules/src/test/java/org/archive/modules/canonicalize/StripSessionIDsTest.java create mode 100644 modules/src/test/java/org/archive/modules/canonicalize/StripUserinfoRuleTest.java create mode 100644 modules/src/test/java/org/archive/modules/canonicalize/StripWWWNRuleTest.java create mode 100644 modules/src/test/java/org/archive/modules/canonicalize/StripWWWRuleTest.java create mode 100644 modules/src/test/java/org/archive/modules/credential/CredentialStoreTest.java create mode 100644 modules/src/test/java/org/archive/modules/credential/HtmlFormCredentialTest.java create mode 100644 modules/src/test/java/org/archive/modules/credential/Rfc2617CredentialTest.java create mode 100644 modules/src/test/java/org/archive/modules/deciderules/DecideRuleSequenceTest.java create mode 100644 modules/src/test/java/org/archive/modules/deciderules/HasViaDecideRuleTest.java create mode 100644 modules/src/test/java/org/archive/modules/deciderules/HopsPathMatchesRegExpDecideRuleTest.java create mode 100644 modules/src/test/java/org/archive/modules/extractor/AggressiveExtractorHTMLTest.java create mode 100644 modules/src/test/java/org/archive/modules/extractor/ExtractorCSSTest.java create mode 100644 modules/src/test/java/org/archive/modules/extractor/ExtractorDOCTest.java create mode 100644 modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java create mode 100644 modules/src/test/java/org/archive/modules/extractor/ExtractorHTTPTest.java create mode 100644 modules/src/test/java/org/archive/modules/extractor/ExtractorImpliedURITest.java create mode 100644 modules/src/test/java/org/archive/modules/extractor/ExtractorJSTest.java create mode 100644 modules/src/test/java/org/archive/modules/extractor/ExtractorPDFTest.java create mode 100644 modules/src/test/java/org/archive/modules/extractor/ExtractorSWFTest.java create mode 100644 modules/src/test/java/org/archive/modules/extractor/ExtractorURITest.java create mode 100644 modules/src/test/java/org/archive/modules/extractor/ExtractorUniversalTest.java create mode 100644 modules/src/test/java/org/archive/modules/extractor/ExtractorXMLTest.java create mode 100644 modules/src/test/java/org/archive/modules/extractor/HTTPContentDigestTest.java create mode 100644 modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java create mode 100644 modules/src/test/java/org/archive/modules/extractor/UnitTestUriLoggerModule.java create mode 100644 modules/src/test/java/org/archive/modules/fetcher/FetchDNSTest.java create mode 100644 modules/src/test/java/org/archive/modules/fetcher/FetchFTPTest.java create mode 100644 modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java create mode 100644 modules/src/test/java/org/archive/modules/net/CrawlHostTest.java create mode 100644 modules/src/test/java/org/archive/modules/net/CrawlServerTest.java create mode 100644 modules/src/test/java/org/archive/modules/net/RobotsHonoringPolicyTest.java create mode 100644 modules/src/test/java/org/archive/modules/net/RobotstxtTest.java create mode 100644 modules/src/test/java/org/archive/modules/net/ServerCacheTest.java create mode 100644 modules/src/test/java/org/archive/modules/recrawl/FetchHistoryProcessorTest.java create mode 100644 modules/src/test/java/org/archive/modules/recrawl/PersistLoadProcessorTest.java create mode 100644 modules/src/test/java/org/archive/modules/recrawl/PersistLogProcessorTest.java create mode 100644 modules/src/test/java/org/archive/modules/recrawl/PersistStoreProcessorTest.java create mode 100644 modules/src/test/java/org/archive/modules/seeds/SeedFileIteratorTest.java create mode 100644 modules/src/test/java/org/archive/modules/writer/ARCWriterProcessorTest.java create mode 100644 modules/src/test/java/org/archive/modules/writer/MirrorWriterProcessorTest.java create mode 100644 modules/src/test/java/org/archive/modules/writer/WARCWriterProcessorTest.java create mode 100644 pom.xml diff --git a/.classpath b/.classpath new file mode 100644 index 00000000..924b6f24 --- /dev/null +++ b/.classpath @@ -0,0 +1,63 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.project b/.project new file mode 100644 index 00000000..a3c7cfba --- /dev/null +++ b/.project @@ -0,0 +1,17 @@ + + + crawler + + + + + + org.eclipse.jdt.core.javabuilder + + + + + + org.eclipse.jdt.core.javanature + + diff --git a/commons/.classpath b/commons/.classpath new file mode 100644 index 00000000..9983c3c8 --- /dev/null +++ b/commons/.classpath @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/commons/.project b/commons/.project new file mode 100644 index 00000000..92929408 --- /dev/null +++ b/commons/.project @@ -0,0 +1,14 @@ + + commons + Heritrix is the Internet Archive's open-source, extensible, web-scale, + archival-quality web crawler project. + + + + org.eclipse.jdt.core.javabuilder + + + + org.eclipse.jdt.core.javanature + + \ No newline at end of file diff --git a/commons/.settings/org.eclipse.jdt.core.prefs b/commons/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 00000000..abddea94 --- /dev/null +++ b/commons/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,5 @@ +#Wed Apr 11 10:57:07 PDT 2007 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.5 +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.source=1.5 +org.eclipse.jdt.core.compiler.compliance=1.5 diff --git a/commons/pom.xml b/commons/pom.xml new file mode 100644 index 00000000..dae46dde --- /dev/null +++ b/commons/pom.xml @@ -0,0 +1,321 @@ + + + + + org.archive + heritrix + 3.0.0-SNAPSHOT + + 4.0.0 + org.archive.heritrix + commons + jar + Heritrix: 'commons' subproject (utility classes) + + The Archive Commons Code Libraries project contains general Java utility + libraries, as used by the Heritrix crawler and other projects. + + + + + + true + always + warn + + + true + never + fail + + oracleReleases + Oracle Released Java Packages + http://download.oracle.com/maven + default + + + + + true + daily + warn + + + true + never + fail + + internetarchive + Internet Archive Maven Repository + http://builds.archive.org:8080/maven2 + default + + + + + + + + + + org.archive.overlays + archive-overlay-commons-httpclient + 3.1 + compile + + + org.archive.overlays + archive-overlay-commons-pool + 1.3 + compile + + + + + com.sleepycat + je + 3.3.74 + + + commons-lang + commons-lang + 2.3 + compile + + + commons-logging + commons-logging + 1.0.4 + compile + + + commons-net + commons-net + 1.4.1 + compile + + + commons-io + commons-io + 1.4 + compile + + + commons-codec + commons-codec + 1.3 + compile + + + commons-collections + commons-collections + 3.1 + compile + + + commons-cli + commons-cli + 1.1 + compile + + + net.htmlparser.jericho + jericho-html + 2.5 + compile + + + org.dnsjava + dnsjava + 2.0.3 + compile + + + + poi + poi + 2.5.1 + compile + + + poi + poi-scratchpad + 2.5.1-final-20040804 + compile + + + itext + itext + 1.3 + compile + + + junit + junit + 3.8.2 + compile + + + fastutil + fastutil + 5.0.7 + compile + + + org.gnu.inet + libidn + 0.6.5 + compile + + + net.java.dev.jets3t + jets3t + 0.5.0 + compile + + + it.unimi.dsi + mg4j + 1.0.1 + compile + + + com.anotherbigidea + javaswf + CVS-SNAPSHOT-1 + compile + + + org.springframework + spring-core + 2.5.6 + + + org.springframework + spring-beans + 2.5.6 + + + org.springframework + spring-context + 2.5.6 + + + + + + src/main/resources + true + + + + + + + + org.apache.maven.plugins + maven-resources-plugin + + UTF-8 + + + + + maven-antrun-plugin + + + generate-timestamp-properties-file + process-resources + run + + + + + + + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.5 + 1.5 + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + false + + **/Test*.java + + + + + + diff --git a/commons/src/main/java/org/apache/commons/httpclient/Cookie.java b/commons/src/main/java/org/apache/commons/httpclient/Cookie.java new file mode 100644 index 00000000..49984fae --- /dev/null +++ b/commons/src/main/java/org/apache/commons/httpclient/Cookie.java @@ -0,0 +1,575 @@ +/* + * $Header: /home/jerenkrantz/tmp/commons/commons-convert/cvs/home/cvs/jakarta-commons//httpclient/src/java/org/apache/commons/httpclient/Cookie.java,v 1.44 2004/06/05 16:49:20 olegk Exp $ + * $Revision$ + * $Date$ + * + * ==================================================================== + * + * Copyright 1999-2004 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ + +package org.apache.commons.httpclient; + +import java.io.Serializable; +import java.text.RuleBasedCollator; +import java.util.Comparator; +import java.util.Date; +import java.util.Locale; + +import org.apache.commons.httpclient.cookie.CookiePolicy; +import org.apache.commons.httpclient.cookie.CookieSpec; +import org.apache.commons.httpclient.util.LangUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + *

+ * HTTP "magic-cookie" represents a piece of state information + * that the HTTP agent and the target server can exchange to maintain + * a session. + *

+ * + * @author B.C. Holmes + * @author Park, Sung-Gu + * @author Doug Sale + * @author Rod Waldhoff + * @author dIon Gillard + * @author Sean C. Sullivan + * @author John Evans + * @author Marc A. Saegesser + * @author Oleg Kalnichevski + * @author Mike Bowler + * + * @version $Revision$ $Date$ + */ +@SuppressWarnings({"serial","unchecked"}) // <- HERITRIX CHANGE +public class Cookie extends NameValuePair implements Serializable, Comparator { + + // ----------------------------------------------------------- Constructors + + /** + * Default constructor. Creates a blank cookie + */ + + public Cookie() { + this(null, "noname", null, null, null, false); + } + + /** + * Creates a cookie with the given name, value and domain attribute. + * + * @param name the cookie name + * @param value the cookie value + * @param domain the domain this cookie can be sent to + */ + public Cookie(String domain, String name, String value) { + this(domain, name, value, null, null, false); + } + + /** + * Creates a cookie with the given name, value, domain attribute, + * path attribute, expiration attribute, and secure attribute + * + * @param name the cookie name + * @param value the cookie value + * @param domain the domain this cookie can be sent to + * @param path the path prefix for which this cookie can be sent + * @param expires the {@link Date} at which this cookie expires, + * or null if the cookie expires at the end + * of the session + * @param secure if true this cookie can only be sent over secure + * connections + * @throws IllegalArgumentException If cookie name is null or blank, + * cookie name contains a blank, or cookie name starts with character $ + * + */ + public Cookie(String domain, String name, String value, + String path, Date expires, boolean secure) { + + super(name, value); + LOG.trace("enter Cookie(String, String, String, String, Date, boolean)"); + if (name == null) { + throw new IllegalArgumentException("Cookie name may not be null"); + } + if (name.trim().equals("")) { + throw new IllegalArgumentException("Cookie name may not be blank"); + } + this.setPath(path); + this.setDomain(domain); + this.setExpiryDate(expires); + this.setSecure(secure); + } + + /** + * Creates a cookie with the given name, value, domain attribute, + * path attribute, maximum age attribute, and secure attribute + * + * @param name the cookie name + * @param value the cookie value + * @param domain the domain this cookie can be sent to + * @param path the path prefix for which this cookie can be sent + * @param maxAge the number of seconds for which this cookie is valid. + * maxAge is expected to be a non-negative number. + * -1 signifies that the cookie should never expire. + * @param secure if true this cookie can only be sent over secure + * connections + */ + public Cookie(String domain, String name, String value, String path, + int maxAge, boolean secure) { + + this(domain, name, value, path, null, secure); + if (maxAge < -1) { + throw new IllegalArgumentException("Invalid max age: " + Integer.toString(maxAge)); + } + if (maxAge >= 0) { + setExpiryDate(new Date(System.currentTimeMillis() + maxAge * 1000L)); + } + } + + /** + * Returns the comment describing the purpose of this cookie, or + * null if no such comment has been defined. + * + * @return comment + * + * @see #setComment(String) + */ + public String getComment() { + return cookieComment; + } + + /** + * If a user agent (web browser) presents this cookie to a user, the + * cookie's purpose will be described using this comment. + * + * @param comment + * + * @see #getComment() + */ + public void setComment(String comment) { + cookieComment = comment; + } + + /** + * Returns the expiration {@link Date} of the cookie, or null + * if none exists. + *

Note: the object returned by this method is + * considered immutable. Changing it (e.g. using setTime()) could result + * in undefined behaviour. Do so at your peril.

+ * @return Expiration {@link Date}, or null. + * + * @see #setExpiryDate(java.util.Date) + * + */ + public Date getExpiryDate() { + return cookieExpiryDate; + } + + /** + * Sets expiration date. + *

Note: the object returned by this method is considered + * immutable. Changing it (e.g. using setTime()) could result in undefined + * behaviour. Do so at your peril.

+ * + * @param expiryDate the {@link Date} after which this cookie is no longer valid. + * + * @see #getExpiryDate + * + */ + public void setExpiryDate (Date expiryDate) { + cookieExpiryDate = expiryDate; + } + + + /** + * Returns false if the cookie should be discarded at the end + * of the "session"; true otherwise. + * + * @return false if the cookie should be discarded at the end + * of the "session"; true otherwise + */ + public boolean isPersistent() { + return (null != cookieExpiryDate); + } + + + /** + * Returns domain attribute of the cookie. + * + * @return the value of the domain attribute + * + * @see #setDomain(java.lang.String) + */ + public String getDomain() { + return cookieDomain; + } + + /** + * Sets the domain attribute. + * + * @param domain The value of the domain attribute + * + * @see #getDomain + */ + public void setDomain(String domain) { + if (domain != null) { + int ndx = domain.indexOf(":"); + if (ndx != -1) { + domain = domain.substring(0, ndx); + } + cookieDomain = domain.toLowerCase(); + } + } + + + /** + * Returns the path attribute of the cookie + * + * @return The value of the path attribute. + * + * @see #setPath(java.lang.String) + */ + public String getPath() { + return cookiePath; + } + + /** + * Sets the path attribute. + * + * @param path The value of the path attribute + * + * @see #getPath + * + */ + public void setPath(String path) { + cookiePath = path; + } + + /** + * @return true if this cookie should only be sent over secure connections. + * @see #setSecure(boolean) + */ + public boolean getSecure() { + return isSecure; + } + + /** + * Sets the secure attribute of the cookie. + *

+ * When true the cookie should only be sent + * using a secure protocol (https). This should only be set when + * the cookie's originating server used a secure protocol to set the + * cookie's value. + * + * @param secure The value of the secure attribute + * + * @see #getSecure() + */ + public void setSecure (boolean secure) { + isSecure = secure; + } + + /** + * Returns the version of the cookie specification to which this + * cookie conforms. + * + * @return the version of the cookie. + * + * @see #setVersion(int) + * + */ + public int getVersion() { + return cookieVersion; + } + + /** + * Sets the version of the cookie specification to which this + * cookie conforms. + * + * @param version the version of the cookie. + * + * @see #getVersion + */ + public void setVersion(int version) { + cookieVersion = version; + } + + /** + * Returns true if this cookie has expired. + * + * @return true if the cookie has expired. + */ + public boolean isExpired() { + return (cookieExpiryDate != null + && cookieExpiryDate.getTime() <= System.currentTimeMillis()); + } + + /** + * Returns true if this cookie has expired according to the time passed in. + * + * @param now The current time. + * + * @return true if the cookie expired. + */ + public boolean isExpired(Date now) { + return (cookieExpiryDate != null + && cookieExpiryDate.getTime() <= now.getTime()); + } + + + /** + * Indicates whether the cookie had a path specified in a + * path attribute of the Set-Cookie header. This value + * is important for generating the Cookie header because + * some cookie specifications require that the Cookie header + * should only include a path attribute if the cookie's path + * was specified in the Set-Cookie header. + * + * @param value true if the cookie's path was explicitly + * set, false otherwise. + * + * @see #isPathAttributeSpecified + */ + public void setPathAttributeSpecified(boolean value) { + hasPathAttribute = value; + } + + /** + * Returns true if cookie's path was set via a path attribute + * in the Set-Cookie header. + * + * @return value true if the cookie's path was explicitly + * set, false otherwise. + * + * @see #setPathAttributeSpecified + */ + public boolean isPathAttributeSpecified() { + return hasPathAttribute; + } + + /** + * Indicates whether the cookie had a domain specified in a + * domain attribute of the Set-Cookie header. This value + * is important for generating the Cookie header because + * some cookie specifications require that the Cookie header + * should only include a domain attribute if the cookie's domain + * was specified in the Set-Cookie header. + * + * @param value true if the cookie's domain was explicitly + * set, false otherwise. + * + * @see #isDomainAttributeSpecified + */ + public void setDomainAttributeSpecified(boolean value) { + hasDomainAttribute = value; + } + + /** + * Returns true if cookie's domain was set via a domain + * attribute in the Set-Cookie header. + * + * @return value true if the cookie's domain was explicitly + * set, false otherwise. + * + * @see #setDomainAttributeSpecified + */ + public boolean isDomainAttributeSpecified() { + return hasDomainAttribute; + } + + /** + * Returns a hash code in keeping with the + * {@link Object#hashCode} general hashCode contract. + * @return A hash code + */ + public int hashCode() { + int hash = LangUtils.HASH_SEED; + hash = LangUtils.hashCode(hash, this.getName()); + hash = LangUtils.hashCode(hash, this.cookieDomain); + hash = LangUtils.hashCode(hash, this.cookiePath); + return hash; + } + + + /** + * Two cookies are equal if the name, path and domain match. + * @param obj The object to compare against. + * @return true if the two objects are equal. + */ + public boolean equals(Object obj) { + if (obj == null) return false; + if (this == obj) return true; + if (obj instanceof Cookie) { + Cookie that = (Cookie) obj; + return LangUtils.equals(this.getName(), that.getName()) + && LangUtils.equals(this.cookieDomain, that.cookieDomain) + && LangUtils.equals(this.cookiePath, that.cookiePath); + } else { + return false; + } + } + + + /** + * Return a textual representation of the cookie. + * + * @return string. + */ + public String toExternalForm() { + CookieSpec spec = null; + if (getVersion() > 0) { + spec = CookiePolicy.getDefaultSpec(); + } else { + spec = CookiePolicy.getCookieSpec(CookiePolicy.NETSCAPE); + } + return spec.formatCookie(this); + } + + /** + *

Compares two cookies to determine order for cookie header.

+ *

Most specific should be first.

+ *

This method is implemented so a cookie can be used as a comparator for + * a SortedSet of cookies. Specifically it's used above in the + * createCookieHeader method.

+ * @param o1 The first object to be compared + * @param o2 The second object to be compared + * @return See {@link java.util.Comparator#compare(Object,Object)} + */ + public int compare(Object o1, Object o2) { + LOG.trace("enter Cookie.compare(Object, Object)"); + + if (!(o1 instanceof Cookie)) { + throw new ClassCastException(o1.getClass().getName()); + } + if (!(o2 instanceof Cookie)) { + throw new ClassCastException(o2.getClass().getName()); + } + Cookie c1 = (Cookie) o1; + Cookie c2 = (Cookie) o2; + if (c1.getPath() == null && c2.getPath() == null) { + return 0; + } else if (c1.getPath() == null) { + // null is assumed to be "/" + if (c2.getPath().equals(CookieSpec.PATH_DELIM)) { + return 0; + } else { + return -1; + } + } else if (c2.getPath() == null) { + // null is assumed to be "/" + if (c1.getPath().equals(CookieSpec.PATH_DELIM)) { + return 0; + } else { + return 1; + } + } else { + return STRING_COLLATOR.compare(c1.getPath(), c2.getPath()); + } + } + + /** + * Return a textual representation of the cookie. + * + * @return string. + * + * @see #toExternalForm + */ + public String toString() { + return toExternalForm(); + } + +// BEGIN IA/HERITRIX ADDITION + /** + * Create a 'sort key' for this Cookie that will cause it to sort + * alongside other Cookies of the same domain (with or without leading + * '.'). This helps cookie-match checks consider only narrow set of + * possible matches, rather than all cookies. + * + * Only two cookies that are equals() (same domain, path, name) will have + * the same sort key. The '\1' separator character is important in + * conjunction with Cookie.DOMAIN+OVERBOUNDS, allowing keys based on the + * domain plus an extension to define the relevant range in a SortedMap. + * @return String sort key for this cookie + */ + public String getSortKey() { + String domain = getDomain(); + return (domain.startsWith(".")) + ? domain.substring(1) + "\1.\1" + getPath() + "\1" + getName() + : domain + "\1\1" + getPath() + "\1" + getName(); + } +// END IA/HERITRIX ADDITION + + // ----------------------------------------------------- Instance Variables + + /** Comment attribute. */ + private String cookieComment; + + /** Domain attribute. */ + private String cookieDomain; + + /** Expiration {@link Date}. */ + private Date cookieExpiryDate; + + /** Path attribute. */ + private String cookiePath; + + /** My secure flag. */ + private boolean isSecure; + + /** + * Specifies if the set-cookie header included a Path attribute for this + * cookie + */ + private boolean hasPathAttribute = false; + + /** + * Specifies if the set-cookie header included a Domain attribute for this + * cookie + */ + private boolean hasDomainAttribute = false; + + /** The version of the cookie specification I was created from. */ + private int cookieVersion = 0; + + // -------------------------------------------------------------- Constants + + /** + * Collator for Cookie comparisons. Could be replaced with references to + * specific Locales. + */ + private static final RuleBasedCollator STRING_COLLATOR = + (RuleBasedCollator) RuleBasedCollator.getInstance( + new Locale("en", "US", "")); + + /** Log object for this class */ + private static final Log LOG = LogFactory.getLog(Cookie.class); + +// BEGIN IA/HERITRIX ADDITION + /** + * Character which, if appended to end of a domain, will give a + * boundary key that sorts past all Cookie sortKeys for the same + * domain. + */ + public static final char DOMAIN_OVERBOUNDS = '\2'; +// END IA/HERITRIX ADDITION +} + diff --git a/commons/src/main/java/org/apache/commons/httpclient/HttpConnection.java b/commons/src/main/java/org/apache/commons/httpclient/HttpConnection.java new file mode 100644 index 00000000..c9e86a61 --- /dev/null +++ b/commons/src/main/java/org/apache/commons/httpclient/HttpConnection.java @@ -0,0 +1,1406 @@ +/* + * $Header: /home/jerenkrantz/tmp/commons/commons-convert/cvs/home/cvs/jakarta-commons//httpclient/src/java/org/apache/commons/httpclient/HttpConnection.java,v 1.107 2005/01/14 21:30:59 olegk Exp $ + * $Revision$ + * $Date$ + * + * ==================================================================== + * + * Copyright 1999-2004 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ + +package org.apache.commons.httpclient; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InterruptedIOException; +import java.io.OutputStream; +import java.lang.reflect.Method; +import java.net.InetAddress; +import java.net.Socket; +import java.net.SocketException; + +import org.apache.commons.httpclient.params.HttpConnectionParams; +import org.apache.commons.httpclient.protocol.Protocol; +import org.apache.commons.httpclient.protocol.ProtocolSocketFactory; +import org.apache.commons.httpclient.protocol.SecureProtocolSocketFactory; +import org.apache.commons.httpclient.util.EncodingUtil; +import org.apache.commons.httpclient.util.ExceptionUtil; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.archive.util.Recorder; // <- // IA/HERITRIX import + +/** + * An abstraction of an HTTP {@link InputStream} and {@link OutputStream} + * pair, together with the relevant attributes. + *

+ * The following options are set on the socket before getting the input/output + * streams in the {@link #open()} method: + * + * + * + * + * + *
Socket Method + * Sockets Option + * Configuration + *
{@link java.net.Socket#setTcpNoDelay(boolean)} + * SO_NODELAY + * {@link HttpConnectionParams#setTcpNoDelay(boolean)} + *
{@link java.net.Socket#setSoTimeout(int)} + * SO_TIMEOUT + * {@link HttpConnectionParams#setSoTimeout(int)} + *
{@link java.net.Socket#setSendBufferSize(int)} + * SO_SNDBUF + * {@link HttpConnectionParams#setSendBufferSize(int)} + *
{@link java.net.Socket#setReceiveBufferSize(int)} + * SO_RCVBUF + * {@link HttpConnectionParams#setReceiveBufferSize(int)} + *
+ * + * @author Rod Waldhoff + * @author Sean C. Sullivan + * @author Ortwin Glueck + * @author Jeff Dever + * @author Mike Bowler + * @author Oleg Kalnichevski + * @author Michael Becke + * @author Eric E Johnson + * @author Laura Werner + * + * @version $Revision$ $Date$ + */ +@SuppressWarnings("unchecked") // <- HERITRIX CHANGE +public class HttpConnection { + + // ----------------------------------------------------------- Constructors + + /** + * Creates a new HTTP connection for the given host and port. + * + * @param host the host to connect to + * @param port the port to connect to + */ + public HttpConnection(String host, int port) { + this(null, -1, host, null, port, Protocol.getProtocol("http")); + } + + /** + * Creates a new HTTP connection for the given host and port + * using the given protocol. + * + * @param host the host to connect to + * @param port the port to connect to + * @param protocol the protocol to use + */ + public HttpConnection(String host, int port, Protocol protocol) { + this(null, -1, host, null, port, protocol); + } + + /** + * Creates a new HTTP connection for the given host with the virtual + * alias and port using given protocol. + * + * @param host the host to connect to + * @param virtualHost the virtual host requests will be sent to + * @param port the port to connect to + * @param protocol the protocol to use + */ + public HttpConnection(String host, String virtualHost, int port, Protocol protocol) { + this(null, -1, host, virtualHost, port, protocol); + } + + /** + * Creates a new HTTP connection for the given host and port via the + * given proxy host and port using the default protocol. + * + * @param proxyHost the host to proxy via + * @param proxyPort the port to proxy via + * @param host the host to connect to + * @param port the port to connect to + */ + public HttpConnection( + String proxyHost, + int proxyPort, + String host, + int port) { + this(proxyHost, proxyPort, host, null, port, Protocol.getProtocol("http")); + } + + /** + * Creates a new HTTP connection for the given host configuration. + * + * @param hostConfiguration the host/proxy/protocol to use + */ + public HttpConnection(HostConfiguration hostConfiguration) { + this(hostConfiguration.getProxyHost(), + hostConfiguration.getProxyPort(), + hostConfiguration.getHost(), + hostConfiguration.getPort(), + hostConfiguration.getProtocol()); + this.localAddress = hostConfiguration.getLocalAddress(); + } + + /** + * Creates a new HTTP connection for the given host with the virtual + * alias and port via the given proxy host and port using the given + * protocol. + * + * @param proxyHost the host to proxy via + * @param proxyPort the port to proxy via + * @param host the host to connect to. Parameter value must be non-null. + * @param virtualHost No longer applicable. + * @param port the port to connect to + * @param protocol The protocol to use. Parameter value must be non-null. + * + * @deprecated use #HttpConnection(String, int, String, int, Protocol) + */ + public HttpConnection( + String proxyHost, + int proxyPort, + String host, + String virtualHost, + int port, + Protocol protocol) { + this(proxyHost, proxyPort, host, port, protocol); + } + + /** + * Creates a new HTTP connection for the given host with the virtual + * alias and port via the given proxy host and port using the given + * protocol. + * + * @param proxyHost the host to proxy via + * @param proxyPort the port to proxy via + * @param host the host to connect to. Parameter value must be non-null. + * @param port the port to connect to + * @param protocol The protocol to use. Parameter value must be non-null. + */ + public HttpConnection( + String proxyHost, + int proxyPort, + String host, + int port, + Protocol protocol) { + + if (host == null) { + throw new IllegalArgumentException("host parameter is null"); + } + if (protocol == null) { + throw new IllegalArgumentException("protocol is null"); + } + + proxyHostName = proxyHost; + proxyPortNumber = proxyPort; + hostName = host; + portNumber = protocol.resolvePort(port); + protocolInUse = protocol; + } + + // ------------------------------------------ Attribute Setters and Getters + + /** + * Returns the connection socket. + * + * @return the socket. + * + * @since 3.0 + */ + protected Socket getSocket() { + return this.socket; + } + + /** + * Returns the host. + * + * @return the host. + */ + public String getHost() { + return hostName; + } + + /** + * Sets the host to connect to. + * + * @param host the host to connect to. Parameter value must be non-null. + * @throws IllegalStateException if the connection is already open + */ + public void setHost(String host) throws IllegalStateException { + if (host == null) { + throw new IllegalArgumentException("host parameter is null"); + } + assertNotOpen(); + hostName = host; + } + + /** + * Returns the target virtual host. + * + * @return the virtual host. + * + * @deprecated no longer applicable + */ + + public String getVirtualHost() { + return this.hostName; + } + + /** + * Sets the virtual host to target. + * + * @param host the virtual host name that should be used instead of + * physical host name when sending HTTP requests. Virtual host + * name can be set to null if virtual host name is not + * to be used + * + * @throws IllegalStateException if the connection is already open + * + * @deprecated no longer applicable + */ + + public void setVirtualHost(String host) throws IllegalStateException { + assertNotOpen(); + } + + /** + * Returns the port of the host. + * + * If the port is -1 (or less than 0) the default port for + * the current protocol is returned. + * + * @return the port. + */ + public int getPort() { + if (portNumber < 0) { + return isSecure() ? 443 : 80; + } else { + return portNumber; + } + } + + /** + * Sets the port to connect to. + * + * @param port the port to connect to + * + * @throws IllegalStateException if the connection is already open + */ + public void setPort(int port) throws IllegalStateException { + assertNotOpen(); + portNumber = port; + } + + /** + * Returns the proxy host. + * + * @return the proxy host. + */ + public String getProxyHost() { + return proxyHostName; + } + + /** + * Sets the host to proxy through. + * + * @param host the host to proxy through. + * + * @throws IllegalStateException if the connection is already open + */ + public void setProxyHost(String host) throws IllegalStateException { + assertNotOpen(); + proxyHostName = host; + } + + /** + * Returns the port of the proxy host. + * + * @return the proxy port. + */ + public int getProxyPort() { + return proxyPortNumber; + } + + /** + * Sets the port of the host to proxy through. + * + * @param port the port of the host to proxy through. + * + * @throws IllegalStateException if the connection is already open + */ + public void setProxyPort(int port) throws IllegalStateException { + assertNotOpen(); + proxyPortNumber = port; + } + + /** + * Returns true if the connection is established over + * a secure protocol. + * + * @return true if connected over a secure protocol. + */ + public boolean isSecure() { + return protocolInUse.isSecure(); + } + + /** + * Returns the protocol used to establish the connection. + * @return The protocol + */ + public Protocol getProtocol() { + return protocolInUse; + } + + /** + * Sets the protocol used to establish the connection + * + * @param protocol The protocol to use. + * + * @throws IllegalStateException if the connection is already open + */ + public void setProtocol(Protocol protocol) { + assertNotOpen(); + + if (protocol == null) { + throw new IllegalArgumentException("protocol is null"); + } + + protocolInUse = protocol; + + } + + /** + * Return the local address used when creating the connection. + * If null, the default address is used. + * + * @return InetAddress the local address to be used when creating Sockets + */ + public InetAddress getLocalAddress() { + return this.localAddress; + } + + /** + * Set the local address used when creating the connection. + * If unset or null, the default address is used. + * + * @param localAddress the local address to use + */ + public void setLocalAddress(InetAddress localAddress) { + assertNotOpen(); + this.localAddress = localAddress; + } + + /** + * Tests if the connection is open. + * + * @return true if the connection is open + */ + public boolean isOpen() { + return isOpen; + } + + /** + * Closes the connection if stale. + * + * @return true if the connection was stale and therefore closed, + * false otherwise. + * + * @see #isStale() + * + * @since 3.0 + */ + public boolean closeIfStale() throws IOException { + if (isOpen && isStale()) { + LOG.debug("Connection is stale, closing..."); + close(); + return true; + } + return false; + } + + /** + * Tests if stale checking is enabled. + * + * @return true if enabled + * + * @see #isStale() + * + * @deprecated Use {@link HttpConnectionParams#isStaleCheckingEnabled()}, + * {@link HttpConnection#getParams()}. + */ + public boolean isStaleCheckingEnabled() { + return this.params.isStaleCheckingEnabled(); + } + + /** + * Sets whether or not isStale() will be called when testing if this connection is open. + * + *

Setting this flag to false will increase performance when reusing + * connections, but it will also make them less reliable. Stale checking ensures that + * connections are viable before they are used. When set to false some + * method executions will result in IOExceptions and they will have to be retried.

+ * + * @param staleCheckEnabled true to enable isStale() + * + * @see #isStale() + * @see #isOpen() + * + * @deprecated Use {@link HttpConnectionParams#setStaleCheckingEnabled(boolean)}, + * {@link HttpConnection#getParams()}. + */ + public void setStaleCheckingEnabled(boolean staleCheckEnabled) { + this.params.setStaleCheckingEnabled(staleCheckEnabled); + } + + /** + * Determines whether this connection is "stale", which is to say that either + * it is no longer open, or an attempt to read the connection would fail. + * + *

Unfortunately, due to the limitations of the JREs prior to 1.4, it is + * not possible to test a connection to see if both the read and write channels + * are open - except by reading and writing. This leads to a difficulty when + * some connections leave the "write" channel open, but close the read channel + * and ignore the request. This function attempts to ameliorate that + * problem by doing a test read, assuming that the caller will be doing a + * write followed by a read, rather than the other way around. + *

+ * + *

To avoid side-effects, the underlying connection is wrapped by a + * {@link BufferedInputStream}, so although data might be read, what is visible + * to clients of the connection will not change with this call.true if the connection is already closed, or a read would + * fail. + */ + protected boolean isStale() throws IOException { + boolean isStale = true; + if (isOpen) { + // the connection is open, but now we have to see if we can read it + // assume the connection is not stale. + isStale = false; + try { + if (inputStream.available() <= 0) { + try { + socket.setSoTimeout(1); + inputStream.mark(1); + int byteRead = inputStream.read(); + if (byteRead == -1) { + // again - if the socket is reporting all data read, + // probably stale + isStale = true; + } else { + inputStream.reset(); + } + } finally { + socket.setSoTimeout(this.params.getSoTimeout()); + } + } + } catch (InterruptedIOException e) { + if (!ExceptionUtil.isSocketTimeoutException(e)) { + throw e; + } + // aha - the connection is NOT stale - continue on! + } catch (IOException e) { + // oops - the connection is stale, the read or soTimeout failed. + LOG.debug( + "An error occurred while reading from the socket, is appears to be stale", + e + ); + isStale = true; + } + } + + return isStale; + } + + /** + * Returns true if the connection is established via a proxy, + * false otherwise. + * + * @return true if a proxy is used to establish the connection, + * false otherwise. + */ + public boolean isProxied() { + return (!(null == proxyHostName || 0 >= proxyPortNumber)); + } + + /** + * Set the state to keep track of the last response for the last request. + * + *

The connection managers use this to ensure that previous requests are + * properly closed before a new request is attempted. That way, a GET + * request need not be read in its entirety before a new request is issued. + * Instead, this stream can be closed as appropriate.

+ * + * @param inStream The stream associated with an HttpMethod. + */ + public void setLastResponseInputStream(InputStream inStream) { + lastResponseInputStream = inStream; + } + + /** + * Returns the stream used to read the last response's body. + * + *

Clients will generally not need to call this function unless + * using HttpConnection directly, instead of calling {@link HttpClient#executeMethod}. + * For those clients, call this function, and if it returns a non-null stream, + * close the stream before attempting to execute a method. Note that + * calling "close" on the stream returned by this function may close + * the connection if the previous response contained a "Connection: close" header.

+ * + * @return An {@link InputStream} corresponding to the body of the last + * response. + */ + public InputStream getLastResponseInputStream() { + return lastResponseInputStream; + } + + // --------------------------------------------------- Other Public Methods + + /** + * Returns {@link HttpConnectionParams HTTP protocol parameters} associated with this method. + * + * @return HTTP parameters. + * + * @since 3.0 + */ + public HttpConnectionParams getParams() { + return this.params; + } + + /** + * Assigns {@link HttpConnectionParams HTTP protocol parameters} for this method. + * + * @since 3.0 + * + * @see HttpConnectionParams + */ + public void setParams(final HttpConnectionParams params) { + if (params == null) { + throw new IllegalArgumentException("Parameters may not be null"); + } + this.params = params; + } + + /** + * Set the {@link Socket}'s timeout, via {@link Socket#setSoTimeout}. If the + * connection is already open, the SO_TIMEOUT is changed. If no connection + * is open, then subsequent connections will use the timeout value. + *

+ * Note: This is not a connection timeout but a timeout on network traffic! + * + * @param timeout the timeout value + * @throws SocketException - if there is an error in the underlying + * protocol, such as a TCP error. + * + * @deprecated Use {@link HttpConnectionParams#setSoTimeout(int)}, + * {@link HttpConnection#getParams()}. + */ + public void setSoTimeout(int timeout) + throws SocketException, IllegalStateException { + this.params.setSoTimeout(timeout); + if (this.socket != null) { + this.socket.setSoTimeout(timeout); + } + } + + /** + * Sets SO_TIMEOUT value directly on the underlying {@link Socket socket}. + * This method does not change the default read timeout value set via + * {@link HttpConnectionParams}. + * + * @param timeout the timeout value + * @throws SocketException - if there is an error in the underlying + * protocol, such as a TCP error. + * @throws IllegalStateException if not connected + * + * @since 3.0 + */ + public void setSocketTimeout(int timeout) + throws SocketException, IllegalStateException { + assertOpen(); + if (this.socket != null) { + this.socket.setSoTimeout(timeout); + } + } + + /** + * Returns the {@link Socket}'s timeout, via {@link Socket#getSoTimeout}, if the + * connection is already open. If no connection is open, return the value subsequent + * connection will use. + *

+ * Note: This is not a connection timeout but a timeout on network traffic! + * + * @return the timeout value + * + * @deprecated Use {@link HttpConnectionParams#getSoTimeout()}, + * {@link HttpConnection#getParams()}. + */ + public int getSoTimeout() throws SocketException { + return this.params.getSoTimeout(); + } + + /** + * Sets the connection timeout. This is the maximum time that may be spent + * until a connection is established. The connection will fail after this + * amount of time. + * @param timeout The timeout in milliseconds. 0 means timeout is not used. + * + * @deprecated Use {@link HttpConnectionParams#setConnectionTimeout(int)}, + * {@link HttpConnection#getParams()}. + */ + public void setConnectionTimeout(int timeout) { + this.params.setConnectionTimeout(timeout); + } + + /** + * Establishes a connection to the specified host and port + * (via a proxy if specified). + * The underlying socket is created from the {@link ProtocolSocketFactory}. + * + * @throws IOException if an attempt to establish the connection results in an + * I/O error. + */ + public void open() throws IOException { + LOG.trace("enter HttpConnection.open()"); + + final String host = (proxyHostName == null) ? hostName : proxyHostName; + final int port = (proxyHostName == null) ? portNumber : proxyPortNumber; + assertNotOpen(); + + if (LOG.isDebugEnabled()) { + LOG.debug("Open connection to " + host + ":" + port); + } + + try { + if (this.socket == null) { + usingSecureSocket = isSecure() && !isProxied(); + // use the protocol's socket factory unless this is a secure + // proxied connection + ProtocolSocketFactory socketFactory = null; + if (isSecure() && isProxied()) { + Protocol defaultprotocol = Protocol.getProtocol("http"); + socketFactory = defaultprotocol.getSocketFactory(); + } else { + socketFactory = this.protocolInUse.getSocketFactory(); + } + this.socket = socketFactory.createSocket( + host, port, + localAddress, 0, + this.params); + } + + /* + "Nagling has been broadly implemented across networks, + including the Internet, and is generally performed by default + - although it is sometimes considered to be undesirable in + highly interactive environments, such as some client/server + situations. In such cases, nagling may be turned off through + use of the TCP_NODELAY sockets option." */ + + socket.setTcpNoDelay(this.params.getTcpNoDelay()); + socket.setSoTimeout(this.params.getSoTimeout()); + + int linger = this.params.getLinger(); + if (linger >= 0) { + socket.setSoLinger(linger > 0, linger); + } + + int sndBufSize = this.params.getSendBufferSize(); + if (sndBufSize >= 0) { + socket.setSendBufferSize(sndBufSize); + } + int rcvBufSize = this.params.getReceiveBufferSize(); + if (rcvBufSize >= 0) { + socket.setReceiveBufferSize(rcvBufSize); + } + int outbuffersize = socket.getSendBufferSize(); + if ((outbuffersize > 2048) || (outbuffersize <= 0)) { + outbuffersize = 2048; + } + int inbuffersize = socket.getReceiveBufferSize(); + if ((inbuffersize > 2048) || (inbuffersize <= 0)) { + inbuffersize = 2048; + } + + // START IA/HERITRIX change + Recorder httpRecorder = Recorder.getHttpRecorder(); + if (httpRecorder == null || (isSecure() && isProxied())) { + // no recorder, OR defer recording for pre-tunnel leg + inputStream = new BufferedInputStream( + socket.getInputStream(), inbuffersize); + outputStream = new BufferedOutputStream( + socket.getOutputStream(), outbuffersize); + } else { + inputStream = httpRecorder.inputWrap((InputStream) + (new BufferedInputStream(socket.getInputStream(), + inbuffersize))); + outputStream = httpRecorder.outputWrap((OutputStream) + (new BufferedOutputStream(socket.getOutputStream(), + outbuffersize))); + } + // END IA/HERITRIX change + + isOpen = true; + } catch (IOException e) { + // Connection wasn't opened properly + // so close everything out + closeSocketAndStreams(); + throw e; + } + } + + /** + * Instructs the proxy to establish a secure tunnel to the host. The socket will + * be switched to the secure socket. Subsequent communication is done via the secure + * socket. The method can only be called once on a proxied secure connection. + * + * @throws IllegalStateException if connection is not secure and proxied or + * if the socket is already secure. + * @throws IOException if an attempt to establish the secure tunnel results in an + * I/O error. + */ + public void tunnelCreated() throws IllegalStateException, IOException { + LOG.trace("enter HttpConnection.tunnelCreated()"); + + if (!isSecure() || !isProxied()) { + throw new IllegalStateException( + "Connection must be secure " + + "and proxied to use this feature"); + } + + if (usingSecureSocket) { + throw new IllegalStateException("Already using a secure socket"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Secure tunnel to " + this.hostName + ":" + this.portNumber); + } + + SecureProtocolSocketFactory socketFactory = + (SecureProtocolSocketFactory) protocolInUse.getSocketFactory(); + + socket = socketFactory.createSocket(socket, hostName, portNumber, true); + int sndBufSize = this.params.getSendBufferSize(); + if (sndBufSize >= 0) { + socket.setSendBufferSize(sndBufSize); + } + int rcvBufSize = this.params.getReceiveBufferSize(); + if (rcvBufSize >= 0) { + socket.setReceiveBufferSize(rcvBufSize); + } + int outbuffersize = socket.getSendBufferSize(); + if (outbuffersize > 2048) { + outbuffersize = 2048; + } + int inbuffersize = socket.getReceiveBufferSize(); + if (inbuffersize > 2048) { + inbuffersize = 2048; + } + + // START IA/HERITRIX change + Recorder httpRecorder = Recorder.getHttpRecorder(); + if (httpRecorder == null) { + inputStream = new BufferedInputStream(socket.getInputStream(), inbuffersize); + outputStream = new BufferedOutputStream(socket.getOutputStream(), outbuffersize); + } else { + inputStream = httpRecorder.inputWrap((InputStream) + (new BufferedInputStream(socket.getInputStream(), + inbuffersize))); + outputStream = httpRecorder.outputWrap((OutputStream) + (new BufferedOutputStream(socket.getOutputStream(), + outbuffersize))); + } + // END IA/HERITRIX change + + usingSecureSocket = true; + tunnelEstablished = true; + } + + /** + * Indicates if the connection is completely transparent from end to end. + * + * @return true if conncetion is not proxied or tunneled through a transparent + * proxy; false otherwise. + */ + public boolean isTransparent() { + return !isProxied() || tunnelEstablished; + } + + /** + * Flushes the output request stream. This method should be called to + * ensure that data written to the request OutputStream is sent to the server. + * + * @throws IOException if an I/O problem occurs + */ + public void flushRequestOutputStream() throws IOException { + LOG.trace("enter HttpConnection.flushRequestOutputStream()"); + assertOpen(); + outputStream.flush(); + } + + /** + * Returns an {@link OutputStream} suitable for writing the request. + * + * @throws IllegalStateException if the connection is not open + * @throws IOException if an I/O problem occurs + * @return a stream to write the request to + */ + public OutputStream getRequestOutputStream() + throws IOException, IllegalStateException { + LOG.trace("enter HttpConnection.getRequestOutputStream()"); + assertOpen(); + OutputStream out = this.outputStream; + if (Wire.CONTENT_WIRE.enabled()) { + out = new WireLogOutputStream(out, Wire.CONTENT_WIRE); + } + return out; + } + + /** + * Return a {@link InputStream} suitable for reading the response. + * @return InputStream The response input stream. + * @throws IOException If an IO problem occurs + * @throws IllegalStateException If the connection isn't open. + */ + public InputStream getResponseInputStream() + throws IOException, IllegalStateException { + LOG.trace("enter HttpConnection.getResponseInputStream()"); + assertOpen(); + return inputStream; + } + + /** + * Tests if input data avaialble. This method returns immediately + * and does not perform any read operations on the input socket + * + * @return boolean true if input data is available, + * false otherwise. + * + * @throws IOException If an IO problem occurs + * @throws IllegalStateException If the connection isn't open. + */ + public boolean isResponseAvailable() + throws IOException { + LOG.trace("enter HttpConnection.isResponseAvailable()"); + if (this.isOpen) { + return this.inputStream.available() > 0; + } else { + return false; + } + } + + /** + * Tests if input data becomes available within the given period time in milliseconds. + * + * @param timeout The number milliseconds to wait for input data to become available + * @return boolean true if input data is availble, + * false otherwise. + * + * @throws IOException If an IO problem occurs + * @throws IllegalStateException If the connection isn't open. + */ + public boolean isResponseAvailable(int timeout) + throws IOException { + LOG.trace("enter HttpConnection.isResponseAvailable(int)"); + assertOpen(); + boolean result = false; + if (this.inputStream.available() > 0) { + result = true; + } else { + try { + this.socket.setSoTimeout(timeout); + inputStream.mark(1); + int byteRead = inputStream.read(); + if (byteRead != -1) { + inputStream.reset(); + LOG.debug("Input data available"); + result = true; + } else { + LOG.debug("Input data not available"); + } + } catch (InterruptedIOException e) { + if (!ExceptionUtil.isSocketTimeoutException(e)) { + throw e; + } + if (LOG.isDebugEnabled()) { + LOG.debug("Input data not available after " + timeout + " ms"); + } + } finally { + try { + socket.setSoTimeout(this.params.getSoTimeout()); + } catch (IOException ioe) { + LOG.debug("An error ocurred while resetting soTimeout, we will assume that" + + " no response is available.", + ioe); + result = false; + } + } + } + return result; + } + + /** + * Writes the specified bytes to the output stream. + * + * @param data the data to be written + * @throws IllegalStateException if not connected + * @throws IOException if an I/O problem occurs + * @see #write(byte[],int,int) + */ + public void write(byte[] data) + throws IOException, IllegalStateException { + LOG.trace("enter HttpConnection.write(byte[])"); + this.write(data, 0, data.length); + } + + /** + * Writes length bytes in data starting at + * offset to the output stream. + * + * The general contract for + * write(b, off, len) is that some of the bytes in the array b are written + * to the output stream in order; element b[off] is the first byte written + * and b[off+len-1] is the last byte written by this operation. + * + * @param data array containing the data to be written. + * @param offset the start offset in the data. + * @param length the number of bytes to write. + * @throws IllegalStateException if not connected + * @throws IOException if an I/O problem occurs + */ + public void write(byte[] data, int offset, int length) + throws IOException, IllegalStateException { + LOG.trace("enter HttpConnection.write(byte[], int, int)"); + + if (offset < 0) { + throw new IllegalArgumentException("Array offset may not be negative"); + } + if (length < 0) { + throw new IllegalArgumentException("Array length may not be negative"); + } + if (offset + length > data.length) { + throw new IllegalArgumentException("Given offset and length exceed the array length"); + } + assertOpen(); + this.outputStream.write(data, offset, length); + } + + /** + * Writes the specified bytes, followed by "\r\n".getBytes() to the + * output stream. + * + * @param data the bytes to be written + * @throws IllegalStateException if the connection is not open + * @throws IOException if an I/O problem occurs + */ + public void writeLine(byte[] data) + throws IOException, IllegalStateException { + LOG.trace("enter HttpConnection.writeLine(byte[])"); + write(data); + writeLine(); + } + + /** + * Writes "\r\n".getBytes() to the output stream. + * + * @throws IllegalStateException if the connection is not open + * @throws IOException if an I/O problem occurs + */ + public void writeLine() + throws IOException, IllegalStateException { + LOG.trace("enter HttpConnection.writeLine()"); + write(CRLF); + } + + /** + * @deprecated Use {@link #print(String, String)} + * + * Writes the specified String (as bytes) to the output stream. + * + * @param data the string to be written + * @throws IllegalStateException if the connection is not open + * @throws IOException if an I/O problem occurs + */ + public void print(String data) + throws IOException, IllegalStateException { + LOG.trace("enter HttpConnection.print(String)"); + write(EncodingUtil.getBytes(data, "ISO-8859-1")); + } + + /** + * Writes the specified String (as bytes) to the output stream. + * + * @param data the string to be written + * @param charset the charset to use for writing the data + * @throws IllegalStateException if the connection is not open + * @throws IOException if an I/O problem occurs + * + * @since 3.0 + */ + public void print(String data, String charset) + throws IOException, IllegalStateException { + LOG.trace("enter HttpConnection.print(String)"); + write(EncodingUtil.getBytes(data, charset)); + } + + /** + * @deprecated Use {@link #printLine(String, String)} + * + * Writes the specified String (as bytes), followed by + * "\r\n".getBytes() to the output stream. + * + * @param data the data to be written + * @throws IllegalStateException if the connection is not open + * @throws IOException if an I/O problem occurs + */ + public void printLine(String data) + throws IOException, IllegalStateException { + LOG.trace("enter HttpConnection.printLine(String)"); + writeLine(EncodingUtil.getBytes(data, "ISO-8859-1")); + } + + /** + * Writes the specified String (as bytes), followed by + * "\r\n".getBytes() to the output stream. + * + * @param data the data to be written + * @param charset the charset to use for writing the data + * @throws IllegalStateException if the connection is not open + * @throws IOException if an I/O problem occurs + * + * @since 3.0 + */ + public void printLine(String data, String charset) + throws IOException, IllegalStateException { + LOG.trace("enter HttpConnection.printLine(String)"); + writeLine(EncodingUtil.getBytes(data, charset)); + } + + /** + * Writes "\r\n".getBytes() to the output stream. + * + * @throws IllegalStateException if the connection is not open + * @throws IOException if an I/O problem occurs + */ + public void printLine() + throws IOException, IllegalStateException { + LOG.trace("enter HttpConnection.printLine()"); + writeLine(); + } + + /** + * Reads up to "\n" from the (unchunked) input stream. + * If the stream ends before the line terminator is found, + * the last part of the string will still be returned. + * + * @throws IllegalStateException if the connection is not open + * @throws IOException if an I/O problem occurs + * @return a line from the response + * + * @deprecated use #readLine(String) + */ + public String readLine() throws IOException, IllegalStateException { + LOG.trace("enter HttpConnection.readLine()"); + + assertOpen(); + return HttpParser.readLine(inputStream); + } + + /** + * Reads up to "\n" from the (unchunked) input stream. + * If the stream ends before the line terminator is found, + * the last part of the string will still be returned. + * + * @param charset the charset to use for reading the data + * + * @throws IllegalStateException if the connection is not open + * @throws IOException if an I/O problem occurs + * @return a line from the response + * + * @since 3.0 + */ + public String readLine(final String charset) throws IOException, IllegalStateException { + LOG.trace("enter HttpConnection.readLine()"); + + assertOpen(); + return HttpParser.readLine(inputStream, charset); + } + + /** + * Attempts to shutdown the {@link Socket}'s output, via Socket.shutdownOutput() + * when running on JVM 1.3 or higher. + * + * @deprecated unused + */ + public void shutdownOutput() { + LOG.trace("enter HttpConnection.shutdownOutput()"); + + try { + // Socket.shutdownOutput is a JDK 1.3 + // method. We'll use reflection in case + // we're running in an older VM + Class[] paramsClasses = new Class[0]; + Method shutdownOutput = + socket.getClass().getMethod("shutdownOutput", paramsClasses); + Object[] params = new Object[0]; + shutdownOutput.invoke(socket, params); + } catch (Exception ex) { + LOG.debug("Unexpected Exception caught", ex); + // Ignore, and hope everything goes right + } + // close output stream? + } + + /** + * Closes the socket and streams. + */ + public void close() { + LOG.trace("enter HttpConnection.close()"); + closeSocketAndStreams(); + } + + /** + * Returns the httpConnectionManager. + * @return HttpConnectionManager + */ + public HttpConnectionManager getHttpConnectionManager() { + return httpConnectionManager; + } + + /** + * Sets the httpConnectionManager. + * @param httpConnectionManager The httpConnectionManager to set + */ + public void setHttpConnectionManager(HttpConnectionManager httpConnectionManager) { + this.httpConnectionManager = httpConnectionManager; + } + + /** + * Releases the connection. If the connection is locked or does not have a connection + * manager associated with it, this method has no effect. Note that it is completely safe + * to call this method multiple times. + */ + public void releaseConnection() { + LOG.trace("enter HttpConnection.releaseConnection()"); + if (locked) { + LOG.debug("Connection is locked. Call to releaseConnection() ignored."); + } else if (httpConnectionManager != null) { + LOG.debug("Releasing connection back to connection manager."); + httpConnectionManager.releaseConnection(this); + } else { + LOG.warn("HttpConnectionManager is null. Connection cannot be released."); + } + } + + /** + * Tests if the connection is locked. Locked connections cannot be released. + * An attempt to release a locked connection will have no effect. + * + * @return true if the connection is locked, false otherwise. + * + * @since 3.0 + */ + protected boolean isLocked() { + return locked; + } + + /** + * Locks or unlocks the connection. Locked connections cannot be released. + * An attempt to release a locked connection will have no effect. + * + * @param locked true to lock the connection, false to unlock + * the connection. + * + * @since 3.0 + */ + protected void setLocked(boolean locked) { + this.locked = locked; + } + // ------------------------------------------------------ Protected Methods + + /** + * Closes everything out. + */ + protected void closeSocketAndStreams() { + LOG.trace("enter HttpConnection.closeSockedAndStreams()"); + + isOpen = false; + + // no longer care about previous responses... + lastResponseInputStream = null; + + if (null != outputStream) { + OutputStream temp = outputStream; + outputStream = null; + try { + temp.close(); + } catch (Exception ex) { + LOG.debug("Exception caught when closing output", ex); + // ignored + } + } + + if (null != inputStream) { + InputStream temp = inputStream; + inputStream = null; + try { + temp.close(); + } catch (Exception ex) { + LOG.debug("Exception caught when closing input", ex); + // ignored + } + } + + if (null != socket) { + Socket temp = socket; + socket = null; + try { + temp.close(); + } catch (Exception ex) { + LOG.debug("Exception caught when closing socket", ex); + // ignored + } + } + + tunnelEstablished = false; + usingSecureSocket = false; + } + + /** + * Throws an {@link IllegalStateException} if the connection is already open. + * + * @throws IllegalStateException if connected + */ + protected void assertNotOpen() throws IllegalStateException { + if (isOpen) { + throw new IllegalStateException("Connection is open"); + } + } + + /** + * Throws an {@link IllegalStateException} if the connection is not open. + * + * @throws IllegalStateException if not connected + */ + protected void assertOpen() throws IllegalStateException { + if (!isOpen) { + throw new IllegalStateException("Connection is not open"); + } + } + + /** + * Gets the socket's sendBufferSize. + * + * @return the size of the buffer for the socket OutputStream, -1 if the value + * has not been set and the socket has not been opened + * + * @throws SocketException if an error occurs while getting the socket value + * + * @see Socket#getSendBufferSize() + */ + public int getSendBufferSize() throws SocketException { + if (socket == null) { + return -1; + } else { + return socket.getSendBufferSize(); + } + } + + /** + * Sets the socket's sendBufferSize. + * + * @param sendBufferSize the size to set for the socket OutputStream + * + * @throws SocketException if an error occurs while setting the socket value + * + * @see Socket#setSendBufferSize(int) + * + * @deprecated Use {@link HttpConnectionParams#setSendBufferSize(int)}, + * {@link HttpConnection#getParams()}. + */ + public void setSendBufferSize(int sendBufferSize) throws SocketException { + this.params.setSendBufferSize(sendBufferSize); + } + + // ------------------------------------------------------- Static Variable + + /** "\r\n", as bytes. */ + private static final byte[] CRLF = new byte[] {(byte) 13, (byte) 10}; + + /** Log object for this class. */ + private static final Log LOG = LogFactory.getLog(HttpConnection.class); + + // ----------------------------------------------------- Instance Variables + + /** My host. */ + private String hostName = null; + + /** My port. */ + private int portNumber = -1; + + /** My proxy host. */ + private String proxyHostName = null; + + /** My proxy port. */ + private int proxyPortNumber = -1; + + /** My client Socket. */ + private Socket socket = null; + + /** My InputStream. */ + private InputStream inputStream = null; + + /** My OutputStream. */ + private OutputStream outputStream = null; + + /** An {@link InputStream} for the response to an individual request. */ + private InputStream lastResponseInputStream = null; + + /** Whether or not the connection is connected. */ + protected boolean isOpen = false; + + /** the protocol being used */ + private Protocol protocolInUse; + + /** Collection of HTTP parameters associated with this HTTP connection*/ + private HttpConnectionParams params = new HttpConnectionParams(); + + /** flag to indicate if this connection can be released, if locked the connection cannot be + * released */ + private boolean locked = false; + + /** Whether or not the socket is a secure one. */ + private boolean usingSecureSocket = false; + + /** Whether the connection is open via a secure tunnel or not */ + private boolean tunnelEstablished = false; + + /** the connection manager that created this connection or null */ + private HttpConnectionManager httpConnectionManager; + + /** The local interface on which the connection is created, or null for the default */ + private InetAddress localAddress; +} diff --git a/commons/src/main/java/org/apache/commons/httpclient/HttpMethodBase.java b/commons/src/main/java/org/apache/commons/httpclient/HttpMethodBase.java new file mode 100644 index 00000000..41644817 --- /dev/null +++ b/commons/src/main/java/org/apache/commons/httpclient/HttpMethodBase.java @@ -0,0 +1,2422 @@ +/* + * $Header: /home/jerenkrantz/tmp/commons/commons-convert/cvs/home/cvs/jakarta-commons//httpclient/src/java/org/apache/commons/httpclient/HttpMethodBase.java,v 1.222 2005/01/14 21:16:40 olegk Exp $ + * $Revision$ + * $Date$ + * + * ==================================================================== + * + * Copyright 1999-2004 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ + +package org.apache.commons.httpclient; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InterruptedIOException; +import java.util.Collection; + +import org.apache.commons.httpclient.auth.AuthState; +import org.apache.commons.httpclient.cookie.CookiePolicy; +import org.apache.commons.httpclient.cookie.CookieSpec; +import org.apache.commons.httpclient.cookie.MalformedCookieException; +import org.apache.commons.httpclient.params.HttpMethodParams; +import org.apache.commons.httpclient.protocol.Protocol; +import org.apache.commons.httpclient.util.EncodingUtil; +import org.apache.commons.httpclient.util.ExceptionUtil; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * An abstract base implementation of HttpMethod. + *

+ * At minimum, subclasses will need to override: + *

    + *
  • {@link #getName} to return the approriate name for this method + *
    • + *
+ *

+ * + *

+ * When a method requires additional request headers, subclasses will typically + * want to override: + *

    + *
  • {@link #addRequestHeaders addRequestHeaders(HttpState,HttpConnection)} + * to write those headers + *
    • + *
+ *

+ * + *

+ * When a method expects specific response headers, subclasses may want to + * override: + *

    + *
  • {@link #processResponseHeaders processResponseHeaders(HttpState,HttpConnection)} + * to handle those headers + *
    • + *
+ *

+ * + * + * @author Remy Maucherat + * @author Rodney Waldhoff + * @author Sean C. Sullivan + * @author dIon Gillard + * @author Jeff Dever + * @author Davanum Srinivas + * @author Ortwin Glueck + * @author Eric Johnson + * @author Michael Becke + * @author Oleg Kalnichevski + * @author Mike Bowler + * @author Gary Gregory + * @author Christian Kohlschuetter + * + * @version $Revision$ $Date$ + */ +@SuppressWarnings({"deprecation","unchecked"}) // <- // IA/HERITRIX change +public abstract class HttpMethodBase implements HttpMethod { + + // -------------------------------------------------------------- Constants + + /** Log object for this class. */ + private static final Log LOG = LogFactory.getLog(HttpMethodBase.class); + + // ----------------------------------------------------- Instance variables + + /** Request headers, if any. */ + private HeaderGroup requestHeaders = new HeaderGroup(); + + /** The Status-Line from the response. */ + private StatusLine statusLine = null; + + /** Response headers, if any. */ + private HeaderGroup responseHeaders = new HeaderGroup(); + + /** Response trailer headers, if any. */ + private HeaderGroup responseTrailerHeaders = new HeaderGroup(); + + /** Path of the HTTP method. */ + private String path = null; + + /** Query string of the HTTP method, if any. */ + private String queryString = null; + + /** The response body of the HTTP method, assuming it has not be + * intercepted by a sub-class. */ + private InputStream responseStream = null; + + /** The connection that the response stream was read from. */ + private HttpConnection responseConnection = null; + + /** Buffer for the response */ + private byte[] responseBody = null; + + /** True if the HTTP method should automatically follow HTTP redirects.*/ + private boolean followRedirects = false; + + /** True if the HTTP method should automatically handle + * HTTP authentication challenges. */ + private boolean doAuthentication = true; + + /** HTTP protocol parameters. */ + private HttpMethodParams params = new HttpMethodParams(); + + /** Host authentication state */ + private AuthState hostAuthState = new AuthState(); + + /** Proxy authentication state */ + private AuthState proxyAuthState = new AuthState(); + + /** True if this method has already been executed. */ + private boolean used = false; + + /** Count of how many times did this HTTP method transparently handle + * a recoverable exception. */ + private int recoverableExceptionCount = 0; + + /** the host for this HTTP method, can be null */ + private HttpHost httphost = null; + + /** + * Handles method retries + * + * @deprecated no loner used + */ + private MethodRetryHandler methodRetryHandler; + + /** True if the connection must be closed when no longer needed */ + private boolean connectionCloseForced = false; + + /** Number of milliseconds to wait for 100-contunue response. */ + private static final int RESPONSE_WAIT_TIME_MS = 3000; + + /** HTTP protocol version used for execution of this method. */ + private HttpVersion effectiveVersion = null; + + /** Whether the execution of this method has been aborted */ + private transient boolean aborted = false; + + /** Whether the HTTP request has been transmitted to the target + * server it its entirety */ + private boolean requestSent = false; + + /** Actual cookie policy */ + private CookieSpec cookiespec = null; + + /** Default initial size of the response buffer if content length is unknown. */ + private static final int DEFAULT_INITIAL_BUFFER_SIZE = 4*1024; // 4 kB + + // ----------------------------------------------------------- Constructors + + /** + * No-arg constructor. + */ + public HttpMethodBase() { + } + + /** + * Constructor specifying a URI. + * It is responsibility of the caller to ensure that URI elements + * (path & query parameters) are properly encoded (URL safe). + * + * @param uri either an absolute or relative URI. The URI is expected + * to be URL-encoded + * + * @throws IllegalArgumentException when URI is invalid + * @throws IllegalStateException when protocol of the absolute URI is not recognised + */ + public HttpMethodBase(String uri) + throws IllegalArgumentException, IllegalStateException { + + try { + + // create a URI and allow for null/empty uri values + if (uri == null || uri.equals("")) { + uri = "/"; + } +// BEGIN IA/HERITRIX CHANGES +// setURI(new URI(uri, true)); + setURI(new org.archive.net.LaxURI(uri, true)); +// END IA/HERITRIX CHANGES + } catch (URIException e) { + throw new IllegalArgumentException("Invalid uri '" + + uri + "': " + e.getMessage() + ); + } + } + + // ------------------------------------------- Property Setters and Getters + + /** + * Obtains the name of the HTTP method as used in the HTTP request line, + * for example "GET" or "POST". + * + * @return the name of this method + */ + public abstract String getName(); + + /** + * Returns the URI of the HTTP method + * + * @return The URI + * + * @throws URIException If the URI cannot be created. + * + * @see org.apache.commons.httpclient.HttpMethod#getURI() + */ + public URI getURI() throws URIException { + StringBuffer buffer = new StringBuffer(); + if (this.httphost != null) { + buffer.append(this.httphost.getProtocol().getScheme()); + buffer.append("://"); + buffer.append(this.httphost.getHostName()); + int port = this.httphost.getPort(); + if (port != -1 && port != this.httphost.getProtocol().getDefaultPort()) { + buffer.append(":"); + buffer.append(port); + } + } + buffer.append(this.path); + if (this.queryString != null) { + buffer.append('?'); + buffer.append(this.queryString); + } +// BEGIN IA/HERITRIX CHANGES +// return new URI(buffer.toString(), true); + return new org.archive.net.LaxURI(buffer.toString(), true); +// END IA/HERITRIX CHANGES + } + + /** + * Sets the URI for this method. + * + * @param uri URI to be set + * + * @throws URIException if a URI cannot be set + * + * @since 3.0 + */ + public void setURI(URI uri) throws URIException { + // only set the host if specified by the URI + if (uri.isAbsoluteURI()) { + this.httphost = new HttpHost(uri); + } + // set the path, defaulting to root + setPath( + uri.getPath() == null + ? "/" + : uri.getEscapedPath() + ); + setQueryString(uri.getEscapedQuery()); + } + + /** + * Sets whether or not the HTTP method should automatically follow HTTP redirects + * (status code 302, etc.) + * + * @param followRedirects true if the method will automatically follow redirects, + * false otherwise. + */ + public void setFollowRedirects(boolean followRedirects) { + this.followRedirects = followRedirects; + } + + /** + * Returns true if the HTTP method should automatically follow HTTP redirects + * (status code 302, etc.), false otherwise. + * + * @return true if the method will automatically follow HTTP redirects, + * false otherwise. + */ + public boolean getFollowRedirects() { + return this.followRedirects; + } + + /** Sets whether version 1.1 of the HTTP protocol should be used per default. + * + * @param http11 true to use HTTP/1.1, false to use 1.0 + * + * @deprecated Use {@link HttpMethodParams#setVersion(HttpVersion)} + */ + public void setHttp11(boolean http11) { + if (http11) { + this.params.setVersion(HttpVersion.HTTP_1_1); + } else { + this.params.setVersion(HttpVersion.HTTP_1_0); + } + } + + /** + * Returns true if the HTTP method should automatically handle HTTP + * authentication challenges (status code 401, etc.), false otherwise + * + * @return true if authentication challenges will be processed + * automatically, false otherwise. + * + * @since 2.0 + */ + public boolean getDoAuthentication() { + return doAuthentication; + } + + /** + * Sets whether or not the HTTP method should automatically handle HTTP + * authentication challenges (status code 401, etc.) + * + * @param doAuthentication true to process authentication challenges + * authomatically, false otherwise. + * + * @since 2.0 + */ + public void setDoAuthentication(boolean doAuthentication) { + this.doAuthentication = doAuthentication; + } + + // ---------------------------------------------- Protected Utility Methods + + /** + * Returns true if version 1.1 of the HTTP protocol should be + * used per default, false if version 1.0 should be used. + * + * @return true to use HTTP/1.1, false to use 1.0 + * + * @deprecated Use {@link HttpMethodParams#getVersion()} + */ + public boolean isHttp11() { + return this.params.getVersion().equals(HttpVersion.HTTP_1_1); + } + + /** + * Sets the path of the HTTP method. + * It is responsibility of the caller to ensure that the path is + * properly encoded (URL safe). + * + * @param path the path of the HTTP method. The path is expected + * to be URL-encoded + */ + public void setPath(String path) { + this.path = path; + } + + /** + * Adds the specified request header, NOT overwriting any previous value. + * Note that header-name matching is case insensitive. + * + * @param header the header to add to the request + */ + public void addRequestHeader(Header header) { + LOG.trace("HttpMethodBase.addRequestHeader(Header)"); + + if (header == null) { + LOG.debug("null header value ignored"); + } else { + getRequestHeaderGroup().addHeader(header); + } + } + + /** + * Use this method internally to add footers. + * + * @param footer The footer to add. + */ + public void addResponseFooter(Header footer) { + getResponseTrailerHeaderGroup().addHeader(footer); + } + + /** + * Gets the path of this HTTP method. + * Calling this method after the request has been executed will + * return the actual path, following any redirects automatically + * handled by this HTTP method. + * + * @return the path to request or "/" if the path is blank. + */ + public String getPath() { + return (path == null || path.equals("")) ? "/" : path; + } + + /** + * Sets the query string of this HTTP method. The caller must ensure that the string + * is properly URL encoded. The query string should not start with the question + * mark character. + * + * @param queryString the query string + * + * @see EncodingUtil#formUrlEncode(NameValuePair[], String) + */ + public void setQueryString(String queryString) { + this.queryString = queryString; + } + + /** + * Sets the query string of this HTTP method. The pairs are encoded as UTF-8 characters. + * To use a different charset the parameters can be encoded manually using EncodingUtil + * and set as a single String. + * + * @param params an array of {@link NameValuePair}s to add as query string + * parameters. The name/value pairs will be automcatically + * URL encoded + * + * @see EncodingUtil#formUrlEncode(NameValuePair[], String) + * @see #setQueryString(String) + */ + public void setQueryString(NameValuePair[] params) { + LOG.trace("enter HttpMethodBase.setQueryString(NameValuePair[])"); + queryString = EncodingUtil.formUrlEncode(params, "UTF-8"); + } + + /** + * Gets the query string of this HTTP method. + * + * @return The query string + */ + public String getQueryString() { + return queryString; + } + + /** + * Set the specified request header, overwriting any previous value. Note + * that header-name matching is case-insensitive. + * + * @param headerName the header's name + * @param headerValue the header's value + */ + public void setRequestHeader(String headerName, String headerValue) { + Header header = new Header(headerName, headerValue); + setRequestHeader(header); + } + + /** + * Sets the specified request header, overwriting any previous value. + * Note that header-name matching is case insensitive. + * + * @param header the header + */ + public void setRequestHeader(Header header) { + + Header[] headers = getRequestHeaderGroup().getHeaders(header.getName()); + + for (int i = 0; i < headers.length; i++) { + getRequestHeaderGroup().removeHeader(headers[i]); + } + + getRequestHeaderGroup().addHeader(header); + + } + + /** + * Returns the specified request header. Note that header-name matching is + * case insensitive. null will be returned if either + * headerName is null or there is no matching header for + * headerName. + * + * @param headerName The name of the header to be returned. + * + * @return The specified request header. + * + * @since 3.0 + */ + public Header getRequestHeader(String headerName) { + if (headerName == null) { + return null; + } else { + return getRequestHeaderGroup().getCondensedHeader(headerName); + } + } + + /** + * Returns an array of the requests headers that the HTTP method currently has + * + * @return an array of my request headers. + */ + public Header[] getRequestHeaders() { + return getRequestHeaderGroup().getAllHeaders(); + } + + /** + * @see org.apache.commons.httpclient.HttpMethod#getRequestHeaders(java.lang.String) + */ + public Header[] getRequestHeaders(String headerName) { + return getRequestHeaderGroup().getHeaders(headerName); + } + + /** + * Gets the {@link HeaderGroup header group} storing the request headers. + * + * @return a HeaderGroup + * + * @since 2.0beta1 + */ + protected HeaderGroup getRequestHeaderGroup() { + return requestHeaders; + } + + /** + * Gets the {@link HeaderGroup header group} storing the response trailer headers + * as per RFC 2616 section 3.6.1. + * + * @return a HeaderGroup + * + * @since 2.0beta1 + */ + protected HeaderGroup getResponseTrailerHeaderGroup() { + return responseTrailerHeaders; + } + + /** + * Gets the {@link HeaderGroup header group} storing the response headers. + * + * @return a HeaderGroup + * + * @since 2.0beta1 + */ + protected HeaderGroup getResponseHeaderGroup() { + return responseHeaders; + } + + /** + * @see org.apache.commons.httpclient.HttpMethod#getResponseHeaders(java.lang.String) + * + * @since 3.0 + */ + public Header[] getResponseHeaders(String headerName) { + return getResponseHeaderGroup().getHeaders(headerName); + } + + /** + * Returns the response status code. + * + * @return the status code associated with the latest response. + */ + public int getStatusCode() { + return statusLine.getStatusCode(); + } + + /** + * Provides access to the response status line. + * + * @return the status line object from the latest response. + * @since 2.0 + */ + public StatusLine getStatusLine() { + return statusLine; + } + + /** + * Checks if response data is available. + * @return true if response data is available, false otherwise. + */ + private boolean responseAvailable() { + return (responseBody != null) || (responseStream != null); + } + + /** + * Returns an array of the response headers that the HTTP method currently has + * in the order in which they were read. + * + * @return an array of response headers. + */ + public Header[] getResponseHeaders() { + return getResponseHeaderGroup().getAllHeaders(); + } + + /** + * Gets the response header associated with the given name. Header name + * matching is case insensitive. null will be returned if either + * headerName is null or there is no matching header for + * headerName. + * + * @param headerName the header name to match + * + * @return the matching header + */ + public Header getResponseHeader(String headerName) { + if (headerName == null) { + return null; + } else { + return getResponseHeaderGroup().getCondensedHeader(headerName); + } + } + + + /** + * Return the length (in bytes) of the response body, as specified in a + * Content-Length header. + * + *

+ * Return -1 when the content-length is unknown. + *

+ * + * @return content length, if Content-Length header is available. + * 0 indicates that the request has no body. + * If Content-Length header is not present, the method + * returns -1. + */ + public long getResponseContentLength() { + Header[] headers = getResponseHeaderGroup().getHeaders("Content-Length"); + if (headers.length == 0) { + return -1; + } + if (headers.length > 1) { + LOG.warn("Multiple content-length headers detected"); + } + for (int i = headers.length - 1; i >= 0; i--) { + Header header = headers[i]; + try { + return Long.parseLong(header.getValue()); + } catch (NumberFormatException e) { + if (LOG.isWarnEnabled()) { + LOG.warn("Invalid content-length value: " + e.getMessage()); + } + } + // See if we can have better luck with another header, if present + } + return -1; + } + + + /** + * Returns the response body of the HTTP method, if any, as an array of bytes. + * If response body is not available or cannot be read, returns null + * + * Note: This will cause the entire response body to be buffered in memory. A + * malicious server may easily exhaust all the VM memory. It is strongly + * recommended, to use getResponseAsStream if the content length of the response + * is unknown or resonably large. + * + * @return The response body. + * + * @throws IOException If an I/O (transport) problem occurs while obtaining the + * response body. + */ + public byte[] getResponseBody() throws IOException { + if (this.responseBody == null) { + InputStream instream = getResponseBodyAsStream(); + if (instream != null) { + long contentLength = getResponseContentLength(); + if (contentLength > Integer.MAX_VALUE) { //guard below cast from overflow + throw new IOException("Content too large to be buffered: "+ contentLength +" bytes"); + } + int limit = getParams().getIntParameter(HttpMethodParams.BUFFER_WARN_TRIGGER_LIMIT, 1024*1024); + if ((contentLength == -1) || (contentLength > limit)) { + LOG.warn("Going to buffer response body of large or unknown size. " + +"Using getResponseBodyAsStream instead is recommended."); + } + LOG.debug("Buffering response body"); + ByteArrayOutputStream outstream = new ByteArrayOutputStream( + contentLength > 0 ? (int) contentLength : DEFAULT_INITIAL_BUFFER_SIZE); + byte[] buffer = new byte[4096]; + int len; + while ((len = instream.read(buffer)) > 0) { + outstream.write(buffer, 0, len); + } + outstream.close(); + setResponseStream(null); + this.responseBody = outstream.toByteArray(); + } + } + return this.responseBody; + } + + /** + * Returns the response body of the HTTP method, if any, as an {@link InputStream}. + * If response body is not available, returns null + * + * @return The response body + * + * @throws IOException If an I/O (transport) problem occurs while obtaining the + * response body. + */ + public InputStream getResponseBodyAsStream() throws IOException { + if (responseStream != null) { + return responseStream; + } + if (responseBody != null) { + InputStream byteResponseStream = new ByteArrayInputStream(responseBody); + LOG.debug("re-creating response stream from byte array"); + return byteResponseStream; + } + return null; + } + + /** + * Returns the response body of the HTTP method, if any, as a {@link String}. + * If response body is not available or cannot be read, returns null + * The string conversion on the data is done using the character encoding specified + * in Content-Type header. + * + * Note: This will cause the entire response body to be buffered in memory. A + * malicious server may easily exhaust all the VM memory. It is strongly + * recommended, to use getResponseAsStream if the content length of the response + * is unknown or resonably large. + * + * @return The response body. + * + * @throws IOException If an I/O (transport) problem occurs while obtaining the + * response body. + */ + public String getResponseBodyAsString() throws IOException { + byte[] rawdata = null; + if (responseAvailable()) { + rawdata = getResponseBody(); + } + if (rawdata != null) { + return EncodingUtil.getString(rawdata, getResponseCharSet()); + } else { + return null; + } + } + + /** + * Returns an array of the response footers that the HTTP method currently has + * in the order in which they were read. + * + * @return an array of footers + */ + public Header[] getResponseFooters() { + return getResponseTrailerHeaderGroup().getAllHeaders(); + } + + /** + * Gets the response footer associated with the given name. + * Footer name matching is case insensitive. + * null will be returned if either footerName is + * null or there is no matching footer for footerName + * or there are no footers available. If there are multiple footers + * with the same name, there values will be combined with the ',' separator + * as specified by RFC2616. + * + * @param footerName the footer name to match + * @return the matching footer + */ + public Header getResponseFooter(String footerName) { + if (footerName == null) { + return null; + } else { + return getResponseTrailerHeaderGroup().getCondensedHeader(footerName); + } + } + + /** + * Sets the response stream. + * @param responseStream The new response stream. + */ + protected void setResponseStream(InputStream responseStream) { + this.responseStream = responseStream; + } + + /** + * Returns a stream from which the body of the current response may be read. + * If the method has not yet been executed, if responseBodyConsumed + * has been called, or if the stream returned by a previous call has been closed, + * null will be returned. + * + * @return the current response stream + */ + protected InputStream getResponseStream() { + return responseStream; + } + + /** + * Returns the status text (or "reason phrase") associated with the latest + * response. + * + * @return The status text. + */ + public String getStatusText() { + return statusLine.getReasonPhrase(); + } + + /** + * Defines how strictly HttpClient follows the HTTP protocol specification + * (RFC 2616 and other relevant RFCs). In the strict mode HttpClient precisely + * implements the requirements of the specification, whereas in non-strict mode + * it attempts to mimic the exact behaviour of commonly used HTTP agents, + * which many HTTP servers expect. + * + * @param strictMode true for strict mode, false otherwise + * + * @deprecated Use {@link org.apache.commons.httpclient.params.HttpParams#setParameter(String, Object)} + * to exercise a more granular control over HTTP protocol strictness. + */ + public void setStrictMode(boolean strictMode) { + if (strictMode) { + this.params.makeStrict(); + } else { + this.params.makeLenient(); + } + } + + /** + * @deprecated Use {@link org.apache.commons.httpclient.params.HttpParams#setParameter(String, Object)} + * to exercise a more granular control over HTTP protocol strictness. + * + * @return false + */ + public boolean isStrictMode() { + return false; + } + + /** + * Adds the specified request header, NOT overwriting any previous value. + * Note that header-name matching is case insensitive. + * + * @param headerName the header's name + * @param headerValue the header's value + */ + public void addRequestHeader(String headerName, String headerValue) { + addRequestHeader(new Header(headerName, headerValue)); + } + + /** + * Tests if the connection should be force-closed when no longer needed. + * + * @return true if the connection must be closed + */ + protected boolean isConnectionCloseForced() { + return this.connectionCloseForced; + } + + /** + * Sets whether or not the connection should be force-closed when no longer + * needed. This value should only be set to true in abnormal + * circumstances, such as HTTP protocol violations. + * + * @param b true if the connection must be closed, false + * otherwise. + */ + protected void setConnectionCloseForced(boolean b) { + if (LOG.isDebugEnabled()) { + LOG.debug("Force-close connection: " + b); + } + this.connectionCloseForced = b; + } + + /** + * Tests if the connection should be closed after the method has been executed. + * The connection will be left open when using HTTP/1.1 or if Connection: + * keep-alive header was sent. + * + * @param conn the connection in question + * + * @return boolean true if we should close the connection. + */ + protected boolean shouldCloseConnection(HttpConnection conn) { + // Connection must be closed due to an abnormal circumstance + if (isConnectionCloseForced()) { + LOG.debug("Should force-close connection."); + return true; + } + + Header connectionHeader = null; + // In case being connected via a proxy server + if (!conn.isTransparent()) { + // Check for 'proxy-connection' directive + connectionHeader = responseHeaders.getFirstHeader("proxy-connection"); + } + // In all cases Check for 'connection' directive + // some non-complaint proxy servers send it instread of + // expected 'proxy-connection' directive + if (connectionHeader == null) { + connectionHeader = responseHeaders.getFirstHeader("connection"); + } + // In case the response does not contain any explict connection + // directives, check whether the request does + if (connectionHeader == null) { + connectionHeader = requestHeaders.getFirstHeader("connection"); + } + if (connectionHeader != null) { + if (connectionHeader.getValue().equalsIgnoreCase("close")) { + if (LOG.isDebugEnabled()) { + LOG.debug("Should close connection in response to directive: " + + connectionHeader.getValue()); + } + return true; + } else if (connectionHeader.getValue().equalsIgnoreCase("keep-alive")) { + if (LOG.isDebugEnabled()) { + LOG.debug("Should NOT close connection in response to directive: " + + connectionHeader.getValue()); + } + return false; + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Unknown directive: " + connectionHeader.toExternalForm()); + } + } + } + LOG.debug("Resorting to protocol version default close connection policy"); + // missing or invalid connection header, do the default + if (this.effectiveVersion.greaterEquals(HttpVersion.HTTP_1_1)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Should NOT close connection, using " + this.effectiveVersion.toString()); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Should close connection, using " + this.effectiveVersion.toString()); + } + } + return this.effectiveVersion.lessEquals(HttpVersion.HTTP_1_0); + } + + /** + * Tests if the this method is ready to be executed. + * + * @param state the {@link HttpState state} information associated with this method + * @param conn the {@link HttpConnection connection} to be used + * @throws HttpException If the method is in invalid state. + */ + private void checkExecuteConditions(HttpState state, HttpConnection conn) + throws HttpException { + + if (state == null) { + throw new IllegalArgumentException("HttpState parameter may not be null"); + } + if (conn == null) { + throw new IllegalArgumentException("HttpConnection parameter may not be null"); + } + if (this.aborted) { + throw new IllegalStateException("Method has been aborted"); + } + if (!validate()) { + throw new ProtocolException("HttpMethodBase object not valid"); + } + } + + /** + * Executes this method using the specified HttpConnection and + * HttpState. + * + * @param state {@link HttpState state} information to associate with this + * request. Must be non-null. + * @param conn the {@link HttpConnection connection} to used to execute + * this HTTP method. Must be non-null. + * + * @return the integer status code if one was obtained, or -1 + * + * @throws IOException if an I/O (transport) error occurs + * @throws HttpException if a protocol exception occurs. + */ + public int execute(HttpState state, HttpConnection conn) + throws HttpException, IOException { + + LOG.trace("enter HttpMethodBase.execute(HttpState, HttpConnection)"); + + // this is our connection now, assign it to a local variable so + // that it can be released later + this.responseConnection = conn; + + checkExecuteConditions(state, conn); + this.statusLine = null; + this.connectionCloseForced = false; + + conn.setLastResponseInputStream(null); + + // determine the effective protocol version + if (this.effectiveVersion == null) { + this.effectiveVersion = this.params.getVersion(); + } + + writeRequest(state, conn); + this.requestSent = true; + readResponse(state, conn); + // the method has successfully executed + used = true; + + return statusLine.getStatusCode(); + } + + /** + * Aborts the execution of this method. + * + * @since 3.0 + */ + public void abort() { + if (this.aborted) { + return; + } + this.aborted = true; + HttpConnection conn = this.responseConnection; + if (conn != null) { + conn.close(); + } + } + + /** + * Returns true if the HTTP method has been already {@link #execute executed}, + * but not {@link #recycle recycled}. + * + * @return true if the method has been executed, false otherwise + */ + public boolean hasBeenUsed() { + return used; + } + + /** + * Recycles the HTTP method so that it can be used again. + * Note that all of the instance variables will be reset + * once this method has been called. This method will also + * release the connection being used by this HTTP method. + * + * @see #releaseConnection() + * + * @deprecated no longer supported and will be removed in the future + * version of HttpClient + */ + public void recycle() { + LOG.trace("enter HttpMethodBase.recycle()"); + + releaseConnection(); + + path = null; + followRedirects = false; + doAuthentication = true; + queryString = null; + getRequestHeaderGroup().clear(); + getResponseHeaderGroup().clear(); + getResponseTrailerHeaderGroup().clear(); + statusLine = null; + effectiveVersion = null; + aborted = false; + used = false; + params = new HttpMethodParams(); + responseBody = null; + recoverableExceptionCount = 0; + connectionCloseForced = false; + hostAuthState.invalidate(); + proxyAuthState.invalidate(); + cookiespec = null; + requestSent = false; + } + + /** + * Releases the connection being used by this HTTP method. In particular the + * connection is used to read the response(if there is one) and will be held + * until the response has been read. If the connection can be reused by other + * HTTP methods it is NOT closed at this point. + * + * @since 2.0 + */ + public void releaseConnection() { + try { + if (this.responseStream != null) { + try { + // FYI - this may indirectly invoke responseBodyConsumed. + this.responseStream.close(); + } catch (IOException ignore) { + } + } + } finally { + ensureConnectionRelease(); + } + } + + /** + * Remove the request header associated with the given name. Note that + * header-name matching is case insensitive. + * + * @param headerName the header name + */ + public void removeRequestHeader(String headerName) { + + Header[] headers = getRequestHeaderGroup().getHeaders(headerName); + for (int i = 0; i < headers.length; i++) { + getRequestHeaderGroup().removeHeader(headers[i]); + } + + } + + /** + * Removes the given request header. + * + * @param header the header + */ + public void removeRequestHeader(final Header header) { + if (header == null) { + return; + } + getRequestHeaderGroup().removeHeader(header); + } + + // ---------------------------------------------------------------- Queries + + /** + * Returns true the method is ready to execute, false otherwise. + * + * @return This implementation always returns true. + */ + public boolean validate() { + return true; + } + + + /** + * Returns the actual cookie policy + * + * @param state HTTP state. TODO: to be removed in the future + * + * @return cookie spec + */ + private CookieSpec getCookieSpec(final HttpState state) { + if (this.cookiespec == null) { + int i = state.getCookiePolicy(); + if (i == -1) { + this.cookiespec = CookiePolicy.getCookieSpec(this.params.getCookiePolicy()); + } else { + this.cookiespec = CookiePolicy.getSpecByPolicy(i); + } + this.cookiespec.setValidDateFormats( + (Collection)this.params.getParameter(HttpMethodParams.DATE_PATTERNS)); + } + return this.cookiespec; + } + + /** + * Generates Cookie request headers for those {@link Cookie cookie}s + * that match the given host, port and path. + * + * @param state the {@link HttpState state} information associated with this method + * @param conn the {@link HttpConnection connection} used to execute + * this HTTP method + * + * @throws IOException if an I/O (transport) error occurs. Some transport exceptions + * can be recovered from. + * @throws HttpException if a protocol exception occurs. Usually protocol exceptions + * cannot be recovered from. + */ + protected void addCookieRequestHeader(HttpState state, HttpConnection conn) + throws IOException, HttpException { + + LOG.trace("enter HttpMethodBase.addCookieRequestHeader(HttpState, " + + "HttpConnection)"); + + Header[] cookieheaders = getRequestHeaderGroup().getHeaders("Cookie"); + for (int i = 0; i < cookieheaders.length; i++) { + Header cookieheader = cookieheaders[i]; + if (cookieheader.isAutogenerated()) { + getRequestHeaderGroup().removeHeader(cookieheader); + } + } + + CookieSpec matcher = getCookieSpec(state); + String host = this.params.getVirtualHost(); + if (host == null) { + host = conn.getHost(); + } + // BEGIN IA/HERITRIX CHANGES + Cookie[] cookies = matcher.match(host, conn.getPort(), + getPath(), conn.isSecure(), state.getCookiesMap()); + // END IA/HERITRIX CHANGES + if ((cookies != null) && (cookies.length > 0)) { + if (getParams().isParameterTrue(HttpMethodParams.SINGLE_COOKIE_HEADER)) { + // In strict mode put all cookies on the same header + String s = matcher.formatCookies(cookies); + getRequestHeaderGroup().addHeader(new Header("Cookie", s, true)); + } else { + // In non-strict mode put each cookie on a separate header + for (int i = 0; i < cookies.length; i++) { + String s = matcher.formatCookie(cookies[i]); + getRequestHeaderGroup().addHeader(new Header("Cookie", s, true)); + } + } + } + } + + /** + * Generates Host request header, as long as no Host request + * header already exists. + * + * @param state the {@link HttpState state} information associated with this method + * @param conn the {@link HttpConnection connection} used to execute + * this HTTP method + * + * @throws IOException if an I/O (transport) error occurs. Some transport exceptions + * can be recovered from. + * @throws HttpException if a protocol exception occurs. Usually protocol exceptions + * cannot be recovered from. + */ + protected void addHostRequestHeader(HttpState state, HttpConnection conn) + throws IOException, HttpException { + LOG.trace("enter HttpMethodBase.addHostRequestHeader(HttpState, " + + "HttpConnection)"); + + // Per 19.6.1.1 of RFC 2616, it is legal for HTTP/1.0 based + // applications to send the Host request-header. + // TODO: Add the ability to disable the sending of this header for + // HTTP/1.0 requests. + String host = this.params.getVirtualHost(); + if (host != null) { + LOG.debug("Using virtual host name: " + host); + } else { + host = conn.getHost(); + } + int port = conn.getPort(); + + // Note: RFC 2616 uses the term "internet host name" for what goes on the + // host line. It would seem to imply that host should be blank if the + // host is a number instead of an name. Based on the behavior of web + // browsers, and the fact that RFC 2616 never defines the phrase "internet + // host name", and the bad behavior of HttpClient that follows if we + // send blank, I interpret this as a small misstatement in the RFC, where + // they meant to say "internet host". So IP numbers get sent as host + // entries too. -- Eric Johnson 12/13/2002 + if (LOG.isDebugEnabled()) { + LOG.debug("Adding Host request header"); + } + + //appends the port only if not using the default port for the protocol + if (conn.getProtocol().getDefaultPort() != port) { + host += (":" + port); + } + + setRequestHeader("Host", host); + } + + /** + * Generates Proxy-Connection: Keep-Alive request header when + * communicating via a proxy server. + * + * @param state the {@link HttpState state} information associated with this method + * @param conn the {@link HttpConnection connection} used to execute + * this HTTP method + * + * @throws IOException if an I/O (transport) error occurs. Some transport exceptions + * can be recovered from. + * @throws HttpException if a protocol exception occurs. Usually protocol exceptions + * cannot be recovered from. + */ + protected void addProxyConnectionHeader(HttpState state, + HttpConnection conn) + throws IOException, HttpException { + LOG.trace("enter HttpMethodBase.addProxyConnectionHeader(" + + "HttpState, HttpConnection)"); + if (!conn.isTransparent()) { + if (getRequestHeader("Proxy-Connection") == null) { + addRequestHeader("Proxy-Connection", "Keep-Alive"); + } + } + } + + /** + * Generates all the required request {@link Header header}s + * to be submitted via the given {@link HttpConnection connection}. + * + *

+ * This implementation adds User-Agent, Host, + * Cookie, Authorization, Proxy-Authorization + * and Proxy-Connection headers, when appropriate. + *

+ * + *

+ * Subclasses may want to override this method to to add additional + * headers, and may choose to invoke this implementation (via + * super) to add the "standard" headers. + *

+ * + * @param state the {@link HttpState state} information associated with this method + * @param conn the {@link HttpConnection connection} used to execute + * this HTTP method + * + * @throws IOException if an I/O (transport) error occurs. Some transport exceptions + * can be recovered from. + * @throws HttpException if a protocol exception occurs. Usually protocol exceptions + * cannot be recovered from. + * + * @see #writeRequestHeaders + */ + protected void addRequestHeaders(HttpState state, HttpConnection conn) + throws IOException, HttpException { + LOG.trace("enter HttpMethodBase.addRequestHeaders(HttpState, " + + "HttpConnection)"); + + addUserAgentRequestHeader(state, conn); + addHostRequestHeader(state, conn); + addCookieRequestHeader(state, conn); + addProxyConnectionHeader(state, conn); + } + + /** + * Generates default User-Agent request header, as long as no + * User-Agent request header already exists. + * + * @param state the {@link HttpState state} information associated with this method + * @param conn the {@link HttpConnection connection} used to execute + * this HTTP method + * + * @throws IOException if an I/O (transport) error occurs. Some transport exceptions + * can be recovered from. + * @throws HttpException if a protocol exception occurs. Usually protocol exceptions + * cannot be recovered from. + */ + protected void addUserAgentRequestHeader(HttpState state, + HttpConnection conn) + throws IOException, HttpException { + LOG.trace("enter HttpMethodBase.addUserAgentRequestHeaders(HttpState, " + + "HttpConnection)"); + + if (getRequestHeader("User-Agent") == null) { + String agent = (String)getParams().getParameter(HttpMethodParams.USER_AGENT); + if (agent == null) { + agent = "Jakarta Commons-HttpClient"; + } + setRequestHeader("User-Agent", agent); + } + } + + /** + * Throws an {@link IllegalStateException} if the HTTP method has been already + * {@link #execute executed}, but not {@link #recycle recycled}. + * + * @throws IllegalStateException if the method has been used and not + * recycled + */ + protected void checkNotUsed() throws IllegalStateException { + if (used) { + throw new IllegalStateException("Already used."); + } + } + + /** + * Throws an {@link IllegalStateException} if the HTTP method has not been + * {@link #execute executed} since last {@link #recycle recycle}. + * + * + * @throws IllegalStateException if not used + */ + protected void checkUsed() throws IllegalStateException { + if (!used) { + throw new IllegalStateException("Not Used."); + } + } + + // ------------------------------------------------- Static Utility Methods + + /** + * Generates HTTP request line according to the specified attributes. + * + * @param connection the {@link HttpConnection connection} used to execute + * this HTTP method + * @param name the method name generate a request for + * @param requestPath the path string for the request + * @param query the query string for the request + * @param version the protocol version to use (e.g. HTTP/1.0) + * + * @return HTTP request line + */ + protected static String generateRequestLine(HttpConnection connection, + String name, String requestPath, String query, String version) { + LOG.trace("enter HttpMethodBase.generateRequestLine(HttpConnection, " + + "String, String, String, String)"); + + StringBuffer buf = new StringBuffer(); + // Append method name + buf.append(name); + buf.append(" "); + // Absolute or relative URL? + if (!connection.isTransparent()) { + Protocol protocol = connection.getProtocol(); + buf.append(protocol.getScheme().toLowerCase()); + buf.append("://"); + buf.append(connection.getHost()); + if ((connection.getPort() != -1) + && (connection.getPort() != protocol.getDefaultPort()) + ) { + buf.append(":"); + buf.append(connection.getPort()); + } + } + // Append path, if any + if (requestPath == null) { + buf.append("/"); + } else { + if (!connection.isTransparent() && !requestPath.startsWith("/")) { + buf.append("/"); + } + buf.append(requestPath); + } + // Append query, if any + if (query != null) { + if (query.indexOf("?") != 0) { + buf.append("?"); + } + buf.append(query); + } + // Append protocol + buf.append(" "); + buf.append(version); + buf.append("\r\n"); + + return buf.toString(); + } + + /** + * This method is invoked immediately after + * {@link #readResponseBody(HttpState,HttpConnection)} and can be overridden by + * sub-classes in order to provide custom body processing. + * + *

+ * This implementation does nothing. + *

+ * + * @param state the {@link HttpState state} information associated with this method + * @param conn the {@link HttpConnection connection} used to execute + * this HTTP method + * + * @see #readResponse + * @see #readResponseBody + */ + protected void processResponseBody(HttpState state, HttpConnection conn) { + } + + /** + * This method is invoked immediately after + * {@link #readResponseHeaders(HttpState,HttpConnection)} and can be overridden by + * sub-classes in order to provide custom response headers processing. + + *

+ * This implementation will handle the Set-Cookie and + * Set-Cookie2 headers, if any, adding the relevant cookies to + * the given {@link HttpState}. + *

+ * + * @param state the {@link HttpState state} information associated with this method + * @param conn the {@link HttpConnection connection} used to execute + * this HTTP method + * + * @see #readResponse + * @see #readResponseHeaders + */ + protected void processResponseHeaders(HttpState state, + HttpConnection conn) { + LOG.trace("enter HttpMethodBase.processResponseHeaders(HttpState, " + + "HttpConnection)"); + + Header[] headers = getResponseHeaderGroup().getHeaders("set-cookie2"); + //Only process old style set-cookie headers if new style headres + //are not present + if (headers.length == 0) { + headers = getResponseHeaderGroup().getHeaders("set-cookie"); + } + + CookieSpec parser = getCookieSpec(state); + String host = this.params.getVirtualHost(); + if (host == null) { + host = conn.getHost(); + } + for (int i = 0; i < headers.length; i++) { + Header header = headers[i]; + Cookie[] cookies = null; + try { + cookies = parser.parse( + host, + conn.getPort(), + getPath(), + conn.isSecure(), + header); + } catch (MalformedCookieException e) { + if (LOG.isWarnEnabled()) { + LOG.warn("Invalid cookie header: \"" + + header.getValue() + + "\". " + e.getMessage()); + } + } + if (cookies != null) { + for (int j = 0; j < cookies.length; j++) { + Cookie cookie = cookies[j]; + try { + parser.validate( + host, + conn.getPort(), + getPath(), + conn.isSecure(), + cookie); + state.addCookie(cookie); + if (LOG.isDebugEnabled()) { + LOG.debug("Cookie accepted: \"" + + parser.formatCookie(cookie) + "\""); + } + } catch (MalformedCookieException e) { + if (LOG.isWarnEnabled()) { + LOG.warn("Cookie rejected: \"" + parser.formatCookie(cookie) + + "\". " + e.getMessage()); + } + } + } + } + } + } + + /** + * This method is invoked immediately after + * {@link #readStatusLine(HttpState,HttpConnection)} and can be overridden by + * sub-classes in order to provide custom response status line processing. + * + * @param state the {@link HttpState state} information associated with this method + * @param conn the {@link HttpConnection connection} used to execute + * this HTTP method + * + * @see #readResponse + * @see #readStatusLine + */ + protected void processStatusLine(HttpState state, HttpConnection conn) { + } + + /** + * Reads the response from the given {@link HttpConnection connection}. + * + *

+ * The response is processed as the following sequence of actions: + * + *

    + *
  1. + * {@link #readStatusLine(HttpState,HttpConnection)} is + * invoked to read the request line. + *
    2. + *
  2. + * {@link #processStatusLine(HttpState,HttpConnection)} + * is invoked, allowing the method to process the status line if + * desired. + *
    3. + *
  3. + * {@link #readResponseHeaders(HttpState,HttpConnection)} is invoked to read + * the associated headers. + *
    4. + *
  4. + * {@link #processResponseHeaders(HttpState,HttpConnection)} is invoked, allowing + * the method to process the headers if desired. + *
    5. + *
  5. + * {@link #readResponseBody(HttpState,HttpConnection)} is + * invoked to read the associated body (if any). + *
    6. + *
  6. + * {@link #processResponseBody(HttpState,HttpConnection)} is invoked, allowing the + * method to process the response body if desired. + *
    7. + *
+ * + * Subclasses may want to override one or more of the above methods to to + * customize the processing. (Or they may choose to override this method + * if dramatically different processing is required.) + *

+ * + * @param state the {@link HttpState state} information associated with this method + * @param conn the {@link HttpConnection connection} used to execute + * this HTTP method + * + * @throws IOException if an I/O (transport) error occurs. Some transport exceptions + * can be recovered from. + * @throws HttpException if a protocol exception occurs. Usually protocol exceptions + * cannot be recovered from. + */ + protected void readResponse(HttpState state, HttpConnection conn) + throws IOException, HttpException { + LOG.trace( + "enter HttpMethodBase.readResponse(HttpState, HttpConnection)"); + // Status line & line may have already been received + // if 'expect - continue' handshake has been used + while (this.statusLine == null) { + readStatusLine(state, conn); + processStatusLine(state, conn); + readResponseHeaders(state, conn); + processResponseHeaders(state, conn); + + int status = this.statusLine.getStatusCode(); + if ((status >= 100) && (status < 200)) { + if (LOG.isInfoEnabled()) { + LOG.info("Discarding unexpected response: " + this.statusLine.toString()); + } + this.statusLine = null; + } + } + readResponseBody(state, conn); + processResponseBody(state, conn); + } + + /** + * Read the response body from the given {@link HttpConnection}. + * + *

+ * The current implementation wraps the socket level stream with + * an appropriate stream for the type of response (chunked, content-length, + * or auto-close). If there is no response body, the connection associated + * with the request will be returned to the connection manager. + *

+ * + *

+ * Subclasses may want to override this method to to customize the + * processing. + *

+ * + * @param state the {@link HttpState state} information associated with this method + * @param conn the {@link HttpConnection connection} used to execute + * this HTTP method + * + * @throws IOException if an I/O (transport) error occurs. Some transport exceptions + * can be recovered from. + * @throws HttpException if a protocol exception occurs. Usually protocol exceptions + * cannot be recovered from. + * + * @see #readResponse + * @see #processResponseBody + */ + protected void readResponseBody(HttpState state, HttpConnection conn) + throws IOException, HttpException { + LOG.trace( + "enter HttpMethodBase.readResponseBody(HttpState, HttpConnection)"); + + // assume we are not done with the connection if we get a stream + InputStream stream = readResponseBody(conn); + if (stream == null) { + // done using the connection! + responseBodyConsumed(); + } else { + conn.setLastResponseInputStream(stream); + setResponseStream(stream); + } + } + + /** + * Returns the response body as an {@link InputStream input stream} + * corresponding to the values of the Content-Length and + * Transfer-Encoding headers. If no response body is available + * returns null. + *

+ * + * @see #readResponse + * @see #processResponseBody + * + * @param conn the {@link HttpConnection connection} used to execute + * this HTTP method + * + * @throws IOException if an I/O (transport) error occurs. Some transport exceptions + * can be recovered from. + * @throws HttpException if a protocol exception occurs. Usually protocol exceptions + * cannot be recovered from. + */ + private InputStream readResponseBody(HttpConnection conn) + throws HttpException, IOException { + + LOG.trace("enter HttpMethodBase.readResponseBody(HttpConnection)"); + + responseBody = null; + InputStream is = conn.getResponseInputStream(); + if (Wire.CONTENT_WIRE.enabled()) { + is = new WireLogInputStream(is, Wire.CONTENT_WIRE); + } + boolean canHaveBody = canResponseHaveBody(statusLine.getStatusCode()); + InputStream result = null; + Header transferEncodingHeader = responseHeaders.getFirstHeader("Transfer-Encoding"); + // We use Transfer-Encoding if present and ignore Content-Length. + // RFC2616, 4.4 item number 3 + if (transferEncodingHeader != null) { + + String transferEncoding = transferEncodingHeader.getValue(); + if (!"chunked".equalsIgnoreCase(transferEncoding) + && !"identity".equalsIgnoreCase(transferEncoding)) { + if (LOG.isWarnEnabled()) { + LOG.warn("Unsupported transfer encoding: " + transferEncoding); + } + } + HeaderElement[] encodings = transferEncodingHeader.getElements(); + // The chunked encoding must be the last one applied + // RFC2616, 14.41 + int len = encodings.length; + if ((len > 0) && ("chunked".equalsIgnoreCase(encodings[len - 1].getName()))) { + // if response body is empty + if (conn.isResponseAvailable(conn.getParams().getSoTimeout())) { + result = new ChunkedInputStream(is, this); + } else { + if (getParams().isParameterTrue(HttpMethodParams.STRICT_TRANSFER_ENCODING)) { + throw new ProtocolException("Chunk-encoded body declared but not sent"); + } else { + LOG.warn("Chunk-encoded body missing"); + } + } + } else { + LOG.info("Response content is not chunk-encoded"); + // The connection must be terminated by closing + // the socket as per RFC 2616, 3.6 + setConnectionCloseForced(true); + result = is; + } + } else { + long expectedLength = getResponseContentLength(); + if (expectedLength == -1) { + if (canHaveBody && this.effectiveVersion.greaterEquals(HttpVersion.HTTP_1_1)) { + Header connectionHeader = responseHeaders.getFirstHeader("Connection"); + String connectionDirective = null; + if (connectionHeader != null) { + connectionDirective = connectionHeader.getValue(); + } + if (!"close".equalsIgnoreCase(connectionDirective)) { + LOG.info("Response content length is not known"); + setConnectionCloseForced(true); + } + } + result = is; + } else { + result = new ContentLengthInputStream(is, expectedLength); + } + } + + // See if the response is supposed to have a response body + if (!canHaveBody) { + result = null; + } + // if there is a result - ALWAYS wrap it in an observer which will + // close the underlying stream as soon as it is consumed, and notify + // the watcher that the stream has been consumed. + if (result != null) { + + result = new AutoCloseInputStream( + result, + new ResponseConsumedWatcher() { + public void responseConsumed() { + responseBodyConsumed(); + } + } + ); + } + + return result; + } + + /** + * Reads the response headers from the given {@link HttpConnection connection}. + * + *

+ * Subclasses may want to override this method to to customize the + * processing. + *

+ * + *

+ * "It must be possible to combine the multiple header fields into one + * "field-name: field-value" pair, without changing the semantics of the + * message, by appending each subsequent field-value to the first, each + * separated by a comma." - HTTP/1.0 (4.3) + *

+ * + * @param state the {@link HttpState state} information associated with this method + * @param conn the {@link HttpConnection connection} used to execute + * this HTTP method + * + * @throws IOException if an I/O (transport) error occurs. Some transport exceptions + * can be recovered from. + * @throws HttpException if a protocol exception occurs. Usually protocol exceptions + * cannot be recovered from. + * + * @see #readResponse + * @see #processResponseHeaders + */ + protected void readResponseHeaders(HttpState state, HttpConnection conn) + throws IOException, HttpException { + LOG.trace("enter HttpMethodBase.readResponseHeaders(HttpState," + + "HttpConnection)"); + + getResponseHeaderGroup().clear(); + + Header[] headers = HttpParser.parseHeaders( + conn.getResponseInputStream(), getParams().getHttpElementCharset()); + if (Wire.HEADER_WIRE.enabled()) { + for (int i = 0; i < headers.length; i++) { + Wire.HEADER_WIRE.input(headers[i].toExternalForm()); + } + } + getResponseHeaderGroup().setHeaders(headers); + } + + /** + * Read the status line from the given {@link HttpConnection}, setting my + * {@link #getStatusCode status code} and {@link #getStatusText status + * text}. + * + *

+ * Subclasses may want to override this method to to customize the + * processing. + *

+ * + * @param state the {@link HttpState state} information associated with this method + * @param conn the {@link HttpConnection connection} used to execute + * this HTTP method + * + * @throws IOException if an I/O (transport) error occurs. Some transport exceptions + * can be recovered from. + * @throws HttpException if a protocol exception occurs. Usually protocol exceptions + * cannot be recovered from. + * + * @see StatusLine + */ + protected void readStatusLine(HttpState state, HttpConnection conn) + throws IOException, HttpException { + LOG.trace("enter HttpMethodBase.readStatusLine(HttpState, HttpConnection)"); + + final int maxGarbageLines = getParams(). + getIntParameter(HttpMethodParams.STATUS_LINE_GARBAGE_LIMIT, Integer.MAX_VALUE); + + //read out the HTTP status string + int count = 0; + String s; + do { + s = conn.readLine(getParams().getHttpElementCharset()); + if (s == null && count == 0) { + // The server just dropped connection on us + throw new NoHttpResponseException("The server " + conn.getHost() + + " failed to respond"); + } + if (Wire.HEADER_WIRE.enabled()) { + Wire.HEADER_WIRE.input(s + "\r\n"); + } + if (s != null && StatusLine.startsWithHTTP(s)) { + // Got one + break; + } else if (s == null || count >= maxGarbageLines) { + // Giving up + throw new ProtocolException("The server " + conn.getHost() + + " failed to respond with a valid HTTP response"); + } + count++; + } while(true); + + //create the status line from the status string + statusLine = new StatusLine(s); + + //check for a valid HTTP-Version + String versionStr = statusLine.getHttpVersion(); + if (getParams().isParameterFalse(HttpMethodParams.UNAMBIGUOUS_STATUS_LINE) + && versionStr.equals("HTTP")) { + getParams().setVersion(HttpVersion.HTTP_1_0); + if (LOG.isWarnEnabled()) { + LOG.warn("Ambiguous status line (HTTP protocol version missing):" + + statusLine.toString()); + } + } else { + this.effectiveVersion = HttpVersion.parse(versionStr); + } + + } + + // ------------------------------------------------------ Protected Methods + + /** + *

+ * Sends the request via the given {@link HttpConnection connection}. + *

+ * + *

+ * The request is written as the following sequence of actions: + *

+ * + *
    + *
  1. + * {@link #writeRequestLine(HttpState, HttpConnection)} is invoked to + * write the request line. + *
    2. + *
  2. + * {@link #writeRequestHeaders(HttpState, HttpConnection)} is invoked + * to write the associated headers. + *
    3. + *
  3. + * \r\n is sent to close the head part of the request. + *
    4. + *
  4. + * {@link #writeRequestBody(HttpState, HttpConnection)} is invoked to + * write the body part of the request. + *
    5. + *
+ * + *

+ * Subclasses may want to override one or more of the above methods to to + * customize the processing. (Or they may choose to override this method + * if dramatically different processing is required.) + *

+ * + * @param state the {@link HttpState state} information associated with this method + * @param conn the {@link HttpConnection connection} used to execute + * this HTTP method + * + * @throws IOException if an I/O (transport) error occurs. Some transport exceptions + * can be recovered from. + * @throws HttpException if a protocol exception occurs. Usually protocol exceptions + * cannot be recovered from. + */ + protected void writeRequest(HttpState state, HttpConnection conn) + throws IOException, HttpException { + LOG.trace( + "enter HttpMethodBase.writeRequest(HttpState, HttpConnection)"); + writeRequestLine(state, conn); + writeRequestHeaders(state, conn); + conn.writeLine(); // close head + if (Wire.HEADER_WIRE.enabled()) { + Wire.HEADER_WIRE.output("\r\n"); + } + + HttpVersion ver = getParams().getVersion(); + Header expectheader = getRequestHeader("Expect"); + String expectvalue = null; + if (expectheader != null) { + expectvalue = expectheader.getValue(); + } + if ((expectvalue != null) + && (expectvalue.compareToIgnoreCase("100-continue") == 0)) { + if (ver.greaterEquals(HttpVersion.HTTP_1_1)) { + + // make sure the status line and headers have been sent + conn.flushRequestOutputStream(); + + int readTimeout = conn.getParams().getSoTimeout(); + try { + conn.setSocketTimeout(RESPONSE_WAIT_TIME_MS); + readStatusLine(state, conn); + processStatusLine(state, conn); + readResponseHeaders(state, conn); + processResponseHeaders(state, conn); + + if (this.statusLine.getStatusCode() == HttpStatus.SC_CONTINUE) { + // Discard status line + this.statusLine = null; + LOG.debug("OK to continue received"); + } else { + return; + } + } catch (InterruptedIOException e) { + if (!ExceptionUtil.isSocketTimeoutException(e)) { + throw e; + } + // Most probably Expect header is not recongnized + // Remove the header to signal the method + // that it's okay to go ahead with sending data + removeRequestHeader("Expect"); + LOG.info("100 (continue) read timeout. Resume sending the request"); + } finally { + conn.setSocketTimeout(readTimeout); + } + + } else { + removeRequestHeader("Expect"); + LOG.info("'Expect: 100-continue' handshake is only supported by " + + "HTTP/1.1 or higher"); + } + } + + writeRequestBody(state, conn); + // make sure the entire request body has been sent + conn.flushRequestOutputStream(); + } + + /** + * Writes the request body to the given {@link HttpConnection connection}. + * + *

+ * This method should return true if the request body was actually + * sent (or is empty), or false if it could not be sent for some + * reason. + *

+ * + *

+ * This implementation writes nothing and returns true. + *

+ * + * @param state the {@link HttpState state} information associated with this method + * @param conn the {@link HttpConnection connection} used to execute + * this HTTP method + * + * @return true + * + * @throws IOException if an I/O (transport) error occurs. Some transport exceptions + * can be recovered from. + * @throws HttpException if a protocol exception occurs. Usually protocol exceptions + * cannot be recovered from. + */ + protected boolean writeRequestBody(HttpState state, HttpConnection conn) + throws IOException, HttpException { + return true; + } + + /** + * Writes the request headers to the given {@link HttpConnection connection}. + * + *

+ * This implementation invokes {@link #addRequestHeaders(HttpState,HttpConnection)}, + * and then writes each header to the request stream. + *

+ * + *

+ * Subclasses may want to override this method to to customize the + * processing. + *

+ * + * @param state the {@link HttpState state} information associated with this method + * @param conn the {@link HttpConnection connection} used to execute + * this HTTP method + * + * @throws IOException if an I/O (transport) error occurs. Some transport exceptions + * can be recovered from. + * @throws HttpException if a protocol exception occurs. Usually protocol exceptions + * cannot be recovered from. + * + * @see #addRequestHeaders + * @see #getRequestHeaders + */ + protected void writeRequestHeaders(HttpState state, HttpConnection conn) + throws IOException, HttpException { + LOG.trace("enter HttpMethodBase.writeRequestHeaders(HttpState," + + "HttpConnection)"); + addRequestHeaders(state, conn); + + String charset = getParams().getHttpElementCharset(); + + Header[] headers = getRequestHeaders(); + for (int i = 0; i < headers.length; i++) { + String s = headers[i].toExternalForm(); + if (Wire.HEADER_WIRE.enabled()) { + Wire.HEADER_WIRE.output(s); + } + conn.print(s, charset); + } + } + + /** + * Writes the request line to the given {@link HttpConnection connection}. + * + *

+ * Subclasses may want to override this method to to customize the + * processing. + *

+ * + * @param state the {@link HttpState state} information associated with this method + * @param conn the {@link HttpConnection connection} used to execute + * this HTTP method + * + * @throws IOException if an I/O (transport) error occurs. Some transport exceptions + * can be recovered from. + * @throws HttpException if a protocol exception occurs. Usually protocol exceptions + * cannot be recovered from. + * + * @see #generateRequestLine + */ + protected void writeRequestLine(HttpState state, HttpConnection conn) + throws IOException, HttpException { + LOG.trace( + "enter HttpMethodBase.writeRequestLine(HttpState, HttpConnection)"); + String requestLine = getRequestLine(conn); + if (Wire.HEADER_WIRE.enabled()) { + Wire.HEADER_WIRE.output(requestLine); + } + conn.print(requestLine, getParams().getHttpElementCharset()); + } + + /** + * Returns the request line. + * + * @param conn the {@link HttpConnection connection} used to execute + * this HTTP method + * + * @return The request line. + */ + private String getRequestLine(HttpConnection conn) { + return HttpMethodBase.generateRequestLine(conn, getName(), + getPath(), getQueryString(), this.effectiveVersion.toString()); + } + + /** + * Returns {@link HttpMethodParams HTTP protocol parameters} associated with this method. + * + * @return HTTP parameters. + * + * @since 3.0 + */ + public HttpMethodParams getParams() { + return this.params; + } + + /** + * Assigns {@link HttpMethodParams HTTP protocol parameters} for this method. + * + * @since 3.0 + * + * @see HttpMethodParams + */ + public void setParams(final HttpMethodParams params) { + if (params == null) { + throw new IllegalArgumentException("Parameters may not be null"); + } + this.params = params; + } + + /** + * Returns the HTTP version used with this method (may be null + * if undefined, that is, the method has not been executed) + * + * @return HTTP version. + * + * @since 3.0 + */ + public HttpVersion getEffectiveVersion() { + return this.effectiveVersion; + } + + /** + * Per RFC 2616 section 4.3, some response can never contain a message + * body. + * + * @param status - the HTTP status code + * + * @return true if the message may contain a body, false if it can not + * contain a message body + */ + private static boolean canResponseHaveBody(int status) { + LOG.trace("enter HttpMethodBase.canResponseHaveBody(int)"); + + boolean result = true; + + if ((status >= 100 && status <= 199) || (status == 204) + || (status == 304)) { // NOT MODIFIED + result = false; + } + + return result; + } + + /** + * Returns proxy authentication realm, if it has been used during authentication process. + * Otherwise returns null. + * + * @return proxy authentication realm + * + * @deprecated use #getProxyAuthState() + */ + public String getProxyAuthenticationRealm() { + return this.proxyAuthState.getRealm(); + } + + /** + * Returns authentication realm, if it has been used during authentication process. + * Otherwise returns null. + * + * @return authentication realm + * + * @deprecated use #getHostAuthState() + */ + public String getAuthenticationRealm() { + return this.hostAuthState.getRealm(); + } + + /** + * Returns the character set from the Content-Type header. + * + * @param contentheader The content header. + * @return String The character set. + */ + protected String getContentCharSet(Header contentheader) { + LOG.trace("enter getContentCharSet( Header contentheader )"); + String charset = null; + if (contentheader != null) { + HeaderElement values[] = contentheader.getElements(); + // I expect only one header element to be there + // No more. no less + if (values.length == 1) { + NameValuePair param = values[0].getParameterByName("charset"); + if (param != null) { + // If I get anything "funny" + // UnsupportedEncondingException will result + charset = param.getValue(); + } + } + } + if (charset == null) { + charset = getParams().getContentCharset(); + if (LOG.isDebugEnabled()) { + LOG.debug("Default charset used: " + charset); + } + } + return charset; + } + + + /** + * Returns the character encoding of the request from the Content-Type header. + * + * @return String The character set. + */ + public String getRequestCharSet() { + return getContentCharSet(getRequestHeader("Content-Type")); + } + + + /** + * Returns the character encoding of the response from the Content-Type header. + * + * @return String The character set. + */ + public String getResponseCharSet() { + return getContentCharSet(getResponseHeader("Content-Type")); + } + + /** + * @deprecated no longer used + * + * Returns the number of "recoverable" exceptions thrown and handled, to + * allow for monitoring the quality of the connection. + * + * @return The number of recoverable exceptions handled by the method. + */ + public int getRecoverableExceptionCount() { + return recoverableExceptionCount; + } + + /** + * A response has been consumed. + * + *

The default behavior for this class is to check to see if the connection + * should be closed, and close if need be, and to ensure that the connection + * is returned to the connection manager - if and only if we are not still + * inside the execute call.

+ * + */ + protected void responseBodyConsumed() { + + // make sure this is the initial invocation of the notification, + // ignore subsequent ones. + responseStream = null; + if (responseConnection != null) { + responseConnection.setLastResponseInputStream(null); + + // At this point, no response data should be available. + // If there is data available, regard the connection as being + // unreliable and close it. + + if (shouldCloseConnection(responseConnection)) { + responseConnection.close(); + } else { + try { + if(responseConnection.isResponseAvailable()) { + boolean logExtraInput = + getParams().isParameterTrue(HttpMethodParams.WARN_EXTRA_INPUT); + + if(logExtraInput) { + LOG.warn("Extra response data detected - closing connection"); + } + responseConnection.close(); + } + } + catch (IOException e) { + LOG.warn(e.getMessage()); + responseConnection.close(); + } + } + } + this.connectionCloseForced = false; + ensureConnectionRelease(); + } + + /** + * Insure that the connection is released back to the pool. + */ + private void ensureConnectionRelease() { + if (responseConnection != null) { + responseConnection.releaseConnection(); + responseConnection = null; + } + } + + /** + * Returns the {@link HostConfiguration host configuration}. + * + * @return the host configuration + * + * @deprecated no longer applicable + */ + public HostConfiguration getHostConfiguration() { + HostConfiguration hostconfig = new HostConfiguration(); + hostconfig.setHost(this.httphost); + return hostconfig; + } + /** + * Sets the {@link HostConfiguration host configuration}. + * + * @param hostconfig The hostConfiguration to set + * + * @deprecated no longer applicable + */ + public void setHostConfiguration(final HostConfiguration hostconfig) { + if (hostconfig != null) { + this.httphost = new HttpHost( + hostconfig.getHost(), + hostconfig.getPort(), + hostconfig.getProtocol()); + } else { + this.httphost = null; + } + } + + /** + * Returns the {@link MethodRetryHandler retry handler} for this HTTP method + * + * @return the methodRetryHandler + * + * @deprecated use {@link HttpMethodParams} + */ + public MethodRetryHandler getMethodRetryHandler() { + return methodRetryHandler; + } + + /** + * Sets the {@link MethodRetryHandler retry handler} for this HTTP method + * + * @param handler the methodRetryHandler to use when this method executed + * + * @deprecated use {@link HttpMethodParams} + */ + public void setMethodRetryHandler(MethodRetryHandler handler) { + methodRetryHandler = handler; + } + + /** + * This method is a dirty hack intended to work around + * current (2.0) design flaw that prevents the user from + * obtaining correct status code, headers and response body from the + * preceding HTTP CONNECT method. + * + * TODO: Remove this crap as soon as possible + */ + void fakeResponse( + StatusLine statusline, + HeaderGroup responseheaders, + InputStream responseStream + ) { + // set used so that the response can be read + this.used = true; + this.statusLine = statusline; + this.responseHeaders = responseheaders; + this.responseBody = null; + this.responseStream = responseStream; + } + + /** + * Returns the target host {@link AuthState authentication state} + * + * @return host authentication state + * + * @since 3.0 + */ + public AuthState getHostAuthState() { + return this.hostAuthState; + } + + /** + * Returns the proxy {@link AuthState authentication state} + * + * @return host authentication state + * + * @since 3.0 + */ + public AuthState getProxyAuthState() { + return this.proxyAuthState; + } + + /** + * Tests whether the execution of this method has been aborted + * + * @return true if the execution of this method has been aborted, + * false otherwise + * + * @since 3.0 + */ + public boolean isAborted() { + return this.aborted; + } + + /** + * Returns true if the HTTP has been transmitted to the target + * server in its entirety, false otherwise. This flag can be useful + * for recovery logic. If the request has not been transmitted in its entirety, + * it is safe to retry the failed method. + * + * @return true if the request has been sent, false otherwise + */ + public boolean isRequestSent() { + return this.requestSent; + } + +} diff --git a/commons/src/main/java/org/apache/commons/httpclient/HttpParser.java b/commons/src/main/java/org/apache/commons/httpclient/HttpParser.java new file mode 100644 index 00000000..7ae07c97 --- /dev/null +++ b/commons/src/main/java/org/apache/commons/httpclient/HttpParser.java @@ -0,0 +1,236 @@ +/* + * $Header: /home/jerenkrantz/tmp/commons/commons-convert/cvs/home/cvs/jakarta-commons//httpclient/src/java/org/apache/commons/httpclient/HttpParser.java,v 1.13 2005/01/11 13:57:06 oglueck Exp $ + * $Revision$ + * $Date$ + * + * ==================================================================== + * + * Copyright 1999-2004 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ + +package org.apache.commons.httpclient; + +import java.io.IOException; +import java.io.InputStream; +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; + +import org.apache.commons.httpclient.util.EncodingUtil; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * A utility class for parsing http header values according to + * RFC-2616 Section 4 and 19.3. + * + * @author Michael Becke + * @author Oleg Kalnichevski + * + * @since 2.0beta1 + */ +@SuppressWarnings("unchecked") // <- IA/HERITRIX CHANGE +public class HttpParser { + + /** Log object for this class. */ + private static final Log LOG = LogFactory.getLog(HttpParser.class); + + /** + * Constructor for HttpParser. + */ + private HttpParser() { } + + /** + * Return byte array from an (unchunked) input stream. + * Stop reading when "\n" terminator encountered + * If the stream ends before the line terminator is found, + * the last part of the string will still be returned. + * If no input data available, null is returned. + * + * @param inputStream the stream to read from + * + * @throws IOException if an I/O problem occurs + * @return a byte array from the stream + */ + public static byte[] readRawLine(InputStream inputStream) throws IOException { + LOG.trace("enter HttpParser.readRawLine()"); + + ByteArrayOutputStream buf = new ByteArrayOutputStream(); + int ch; + while ((ch = inputStream.read()) >= 0) { + buf.write(ch); + if (ch == '\n') { // be tolerant (RFC-2616 Section 19.3) + break; + } + } + if (buf.size() == 0) { + return null; + } + return buf.toByteArray(); + } + + /** + * Read up to "\n" from an (unchunked) input stream. + * If the stream ends before the line terminator is found, + * the last part of the string will still be returned. + * If no input data available, null is returned. + * + * @param inputStream the stream to read from + * @param charset charset of HTTP protocol elements + * + * @throws IOException if an I/O problem occurs + * @return a line from the stream + * + * @since 3.0 + */ + public static String readLine(InputStream inputStream, String charset) throws IOException { + LOG.trace("enter HttpParser.readLine(InputStream, String)"); + byte[] rawdata = readRawLine(inputStream); + if (rawdata == null) { + return null; + } + // strip CR and LF from the end + int len = rawdata.length; + int offset = 0; + if (len > 0) { + if (rawdata[len - 1] == '\n') { + offset++; + if (len > 1) { + if (rawdata[len - 2] == '\r') { + offset++; + } + } + } + } + return EncodingUtil.getString(rawdata, 0, len - offset, charset); + } + + /** + * Read up to "\n" from an (unchunked) input stream. + * If the stream ends before the line terminator is found, + * the last part of the string will still be returned. + * If no input data available, null is returned + * + * @param inputStream the stream to read from + * + * @throws IOException if an I/O problem occurs + * @return a line from the stream + * + * @deprecated use #readLine(InputStream, String) + */ + + public static String readLine(InputStream inputStream) throws IOException { + LOG.trace("enter HttpParser.readLine(InputStream)"); + return readLine(inputStream, "US-ASCII"); + } + + /** + * Parses headers from the given stream. Headers with the same name are not + * combined. + * + * @param is the stream to read headers from + * @param charset the charset to use for reading the data + * + * @return an array of headers in the order in which they were parsed + * + * @throws IOException if an IO error occurs while reading from the stream + * @throws HttpException if there is an error parsing a header value + * + * @since 3.0 + */ + public static Header[] parseHeaders(InputStream is, String charset) throws IOException, HttpException { + LOG.trace("enter HeaderParser.parseHeaders(InputStream, String)"); + + ArrayList headers = new ArrayList(); + String name = null; + StringBuffer value = null; + for (; ;) { + String line = HttpParser.readLine(is, charset); + if ((line == null) || (line.trim().length() < 1)) { + break; + } + + // Parse the header name and value + // Check for folded headers first + // Detect LWS-char see HTTP/1.0 or HTTP/1.1 Section 2.2 + // discussion on folded headers + if ((line.charAt(0) == ' ') || (line.charAt(0) == '\t')) { + // we have continuation folded header + // so append value + if (value != null) { + value.append(' '); + value.append(line.trim()); + } + } else { + // make sure we save the previous name,value pair if present + if (name != null) { + headers.add(new Header(name, value.toString())); + } + + // Otherwise we should have normal HTTP header line + // Parse the header name and value + int colon = line.indexOf(":"); + + // START IA/HERITRIX change + // Don't throw an exception if can't parse. We want to keep + // going even though header is bad. Rather, create + // pseudo-header. + if (colon < 0) { + // throw new ProtocolException("Unable to parse header: " + + // line); + name = "HttpClient-Bad-Header-Line-Failed-Parse"; + value = new StringBuffer(line); + + } else { + name = line.substring(0, colon).trim(); + value = new StringBuffer(line.substring(colon + 1).trim()); + } + // END IA/HERITRIX change + } + + } + + // make sure we save the last name,value pair if present + if (name != null) { + headers.add(new Header(name, value.toString())); + } + + return (Header[]) headers.toArray(new Header[headers.size()]); + } + + /** + * Parses headers from the given stream. Headers with the same name are not + * combined. + * + * @param is the stream to read headers from + * + * @return an array of headers in the order in which they were parsed + * + * @throws IOException if an IO error occurs while reading from the stream + * @throws HttpException if there is an error parsing a header value + * + * @deprecated use #parseHeaders(InputStream, String) + */ + public static Header[] parseHeaders(InputStream is) throws IOException, HttpException { + LOG.trace("enter HeaderParser.parseHeaders(InputStream, String)"); + return parseHeaders(is, "US-ASCII"); + } +} diff --git a/commons/src/main/java/org/apache/commons/httpclient/HttpState.java b/commons/src/main/java/org/apache/commons/httpclient/HttpState.java new file mode 100644 index 00000000..a6aa715a --- /dev/null +++ b/commons/src/main/java/org/apache/commons/httpclient/HttpState.java @@ -0,0 +1,717 @@ +/* + * $Header: /home/jerenkrantz/tmp/commons/commons-convert/cvs/home/cvs/jakarta-commons//httpclient/src/java/org/apache/commons/httpclient/HttpState.java,v 1.38 2004/12/20 11:50:54 olegk Exp $ + * $Revision$ + * $Date$ + * + * ==================================================================== + * + * Copyright 1999-2004 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ + +package org.apache.commons.httpclient; + +import java.util.ArrayList; +import java.util.Collection; // <- IA/HERITRIX CHANGE +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.List; +import java.util.Iterator; +import java.util.SortedMap; // <- IA/HERITRIX CHANGE +import java.util.TreeMap; // <- IA/HERITRIX CHANGE + +import org.apache.commons.httpclient.cookie.CookieSpec; +import org.apache.commons.httpclient.cookie.CookiePolicy; +import org.apache.commons.httpclient.auth.AuthScope; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import com.sleepycat.collections.StoredIterator; // <- IA/HERITRIX CHANGE + +/** + *

+ * A container for HTTP attributes that may persist from request + * to request, such as {@link Cookie cookies} and authentication + * {@link Credentials credentials}. + *

+ * + * @author Remy Maucherat + * @author Rodney Waldhoff + * @author Jeff Dever + * @author Sean C. Sullivan + * @author Michael Becke + * @author Oleg Kalnichevski + * @author Mike Bowler + * @author Adrian Sutton + * + * @version $Revision$ $Date$ + * + */ +@SuppressWarnings({"unchecked","unused"}) // <- IA/HERITRIX CHANGE +public class HttpState { + + // ----------------------------------------------------- Instance Variables + + /** + * Map of {@link Credentials credentials} by realm that this + * HTTP state contains. + */ + private HashMap credMap = new HashMap(); + + /** + * Map of {@link Credentials proxy credentials} by realm that this + * HTTP state contains + */ + private HashMap proxyCred = new HashMap(); + +// BEGIN IA/HERITRIX CHANGES +// /** +// * Array of {@link Cookie cookies} that this HTTP state contains. +// */ +// private ArrayList cookiesArrayList = new ArrayList(); + /** + * SortedMap of {@link Cookie cookies} that this HTTP state contains. + */ + private SortedMap cookiesMap = new TreeMap(); +// END IA/HERITRIX CHANGES + + private boolean preemptive = false; + + private int cookiePolicy = -1; + // -------------------------------------------------------- Class Variables + + /** + * The boolean system property name to turn on preemptive authentication. + * @deprecated This field and feature will be removed following HttpClient 3.0. + */ + public static final String PREEMPTIVE_PROPERTY = "httpclient.authentication.preemptive"; + + /** + * The default value for {@link #PREEMPTIVE_PROPERTY}. + * @deprecated This field and feature will be removed following HttpClient 3.0. + */ + public static final String PREEMPTIVE_DEFAULT = "false"; + + /** Log object for this class. */ + private static final Log LOG = LogFactory.getLog(HttpState.class); + + /** + * Default constructor. + */ + public HttpState() { + super(); + } + + // ------------------------------------------------------------- Properties + + /** + * Adds an {@link Cookie HTTP cookie}, replacing any existing equivalent cookies. + * If the given cookie has already expired it will not be added, but existing + * values will still be removed. + * + * @param cookie the {@link Cookie cookie} to be added + * + * @see #addCookies(Cookie[]) + * + */ + public synchronized void addCookie(Cookie cookie) { + LOG.trace("enter HttpState.addCookie(Cookie)"); + +// BEGIN IA/HERITRIX CHANGES +// PRIOR IMPL & COMPARISON HARNESS LEFT COMMENTED OUT FOR TEMPORARY REFERENCE +// Cookie removed1 = null; +// Cookie removed2 = null; + if (cookie != null) { + // first remove any old cookie that is equivalent +// for (Iterator it = cookiesArrayList.iterator(); it.hasNext();) { +// Cookie tmp = (Cookie) it.next(); +// if (cookie.equals(tmp)) { +// it.remove(); +// removed1 = tmp; +// break; +// } +// } + if (!cookie.isExpired()) { +// cookiesArrayList.add(cookie); + cookiesMap.put(cookie.getSortKey(),cookie); + } else { + cookiesMap.remove(cookie.getSortKey()); + } + } +// if(removed1!=null && !removed1.equals(removed2)) { +// System.out.println("addCookie discrepancy"); +// } +// END IA/HERITRIX CHANGES + } + + /** + * Adds an array of {@link Cookie HTTP cookies}. Cookies are added individually and + * in the given array order. If any of the given cookies has already expired it will + * not be added, but existing values will still be removed. + * + * @param cookies the {@link Cookie cookies} to be added + * + * @see #addCookie(Cookie) + * + * + */ + public synchronized void addCookies(Cookie[] cookies) { + LOG.trace("enter HttpState.addCookies(Cookie[])"); + + if (cookies != null) { + for (int i = 0; i < cookies.length; i++) { + this.addCookie(cookies[i]); + } + } + } + + /** + * Returns an array of {@link Cookie cookies} that this HTTP + * state currently contains. + * + * @return an array of {@link Cookie cookies}. + * + * @see #getCookies(String, int, String, boolean) + * + * @deprecated use getCookiesMap() // <- IA/HERITRIX CHANGE + */ + public synchronized Cookie[] getCookies() { + LOG.trace("enter HttpState.getCookies()"); +// BEGIN IA/HERITRIX CHANGES +// PRIOR IMPL & COMPARISON HARNESS LEFT COMMENTED OUT FOR TEMPORARY REFERENCE +// Cookie[] arrayListAnswer = (Cookie[]) (cookiesArrayList.toArray(new Cookie[cookiesArrayList.size()])); + ArrayList arrayableCookies = new ArrayList(); + Iterator iter = cookiesMap.values().iterator(); + while(iter.hasNext()) { + arrayableCookies.add(iter.next()); + } + StoredIterator.close(iter); + Cookie[] mapAnswer = + (Cookie[]) arrayableCookies.toArray(new Cookie[arrayableCookies.size()]); + +// if(cookiesArrayList.size()!=arrayableCookies.size()) { +// System.out.println("discrepancy"); +// } + return mapAnswer; +// END IA/HERITRIX CHANGES + } + +// START IA/HERITRIX ADDITIONS + /** + * Returns a sorted map of {@link Cookie cookies} that this HTTP + * state currently contains. + * + * Any operations on this map should be synchronized with respect + * to this HttpState instance. + * + * @return sorter map of {@link Cookie cookies} + */ + public synchronized SortedMap getCookiesMap() { + return cookiesMap; + } + + /** + * Replace the standard sorted map with an external implemenations + * (such as one backed by persistent store, like BDB's StoredSortedMap.) + * + * @param map alternate sorted map to use to store cookies + */ + public synchronized void setCookiesMap(SortedMap map) { + this.cookiesMap = map; + } +// END IA/HERITRIX ADDITIONS + + /** + * Returns an array of {@link Cookie cookies} in this HTTP + * state that match the given request parameters. + * + * @param domain the request domain + * @param port the request port + * @param path the request path + * @param secure true when using HTTPS + * + * @return an array of {@link Cookie cookies}. + * + * @see #getCookies() + * + * @deprecated use CookieSpec#match(String, int, String, boolean, Cookie) + */ + public synchronized Cookie[] getCookies( + String domain, + int port, + String path, + boolean secure + ) { + LOG.trace("enter HttpState.getCookies(String, int, String, boolean)"); + + CookieSpec matcher = CookiePolicy.getDefaultSpec(); +// BEGIN IA/HERITRIX CHANGES +// PRIOR IMPL & COMPARISON HARNESS LEFT COMMENTED OUT FOR TEMPORARY REFERENCE +// ArrayList list = new ArrayList(cookiesArrayList.size()); +// for (int i = 0, m = cookiesArrayList.size(); i < m; i++) { +// Cookie cookie = (Cookie) (cookiesArrayList.get(i)); +// if (matcher.match(domain, port, path, secure, cookie)) { +// list.add(cookie); +// } +// } +// Cookie[] arrayListAnswer = (Cookie[]) (list.toArray(new Cookie[list.size()])); + Cookie[] mapAnswer = matcher.match(domain,port,path,secure,cookiesMap); + +// if(! (new HashSet(list).equals(new HashSet(Arrays.asList(mapAnswer))))) { +// System.out.println("discrepancy"); +// } + return mapAnswer; +// END IA/HERITRIX CHANGES + } + + /** + * Removes all of {@link Cookie cookies} in this HTTP state + * that have expired according to the current system time. + * + * @see #purgeExpiredCookies(java.util.Date) + * + */ + public synchronized boolean purgeExpiredCookies() { + LOG.trace("enter HttpState.purgeExpiredCookies()"); + return purgeExpiredCookies(new Date()); + } + + /** + * Removes all of {@link Cookie cookies} in this HTTP state + * that have expired by the specified {@link java.util.Date date}. + * + * @param date The {@link java.util.Date date} to compare against. + * + * @return true if any cookies were purged. + * + * @see Cookie#isExpired(java.util.Date) + * + * @see #purgeExpiredCookies() + */ + public synchronized boolean purgeExpiredCookies(Date date) { + LOG.trace("enter HttpState.purgeExpiredCookies(Date)"); +// BEGIN IA/HERITRIX CHANGES +// PRIOR IMPL & COMPARISON HARNESS LEFT COMMENTED OUT FOR TEMPORARY REFERENCE +// boolean arrayRemoved = false; +// Iterator ita = cookiesArrayList.iterator(); +// while (ita.hasNext()) { +// if (((Cookie) (ita.next())).isExpired(date)) { +// ita.remove(); +// arrayRemoved = true; +// } +// } + boolean removed = false; + Iterator it = cookiesMap.values().iterator(); + while (it.hasNext()) { + if (((Cookie) (it.next())).isExpired(date)) { + it.remove(); + removed = true; + } + } + StoredIterator.close(it); +// assert removed == arrayRemoved : "discrepancy" +// END IA/HERITRIX CHANGES + return removed; + } + + + /** + * Returns the current {@link CookiePolicy cookie policy} for this + * HTTP state. + * + * @return The {@link CookiePolicy cookie policy}. + * + * @deprecated Use + * {@link org.apache.commons.httpclient.params.HttpMethodParams#getCookiePolicy()}, + * {@link HttpMethod#getParams()}. + */ + + public int getCookiePolicy() { + return this.cookiePolicy; + } + + + /** + * Defines whether preemptive authentication should be + * attempted. + * + * @param value true if preemptive authentication should be + * attempted, false otherwise. + * + * @deprecated Use + * {@link org.apache.commons.httpclient.params.HttpClientParams#setAuthenticationPreemptive(boolean)}, + * {@link HttpClient#getParams()}. + */ + + public void setAuthenticationPreemptive(boolean value) { + this.preemptive = value; + } + + + /** + * Returns true if preemptive authentication should be + * attempted, false otherwise. + * + * @return boolean flag. + * + * @deprecated Use + * {@link org.apache.commons.httpclient.params.HttpClientParams#isAuthenticationPreemptive()}, + * {@link HttpClient#getParams()}. + */ + + public boolean isAuthenticationPreemptive() { + return this.preemptive; + } + + + /** + * Sets the current {@link CookiePolicy cookie policy} for this HTTP + * state to one of the following supported policies: + * {@link CookiePolicy#COMPATIBILITY}, + * {@link CookiePolicy#NETSCAPE_DRAFT} or + * {@link CookiePolicy#RFC2109}. + * + * @param policy new {@link CookiePolicy cookie policy} + * + * @deprecated + * Use {@link org.apache.commons.httpclient.params.HttpMethodParams#setCookiePolicy(String)}, + * {@link HttpMethod#getParams()}. + */ + + public void setCookiePolicy(int policy) { + this.cookiePolicy = policy; + } + + /** + * Sets the {@link Credentials credentials} for the given authentication + * realm on the given host. The null realm signifies default + * credentials for the given host, which should be used when no + * {@link Credentials credentials} have been explictly supplied for the + * challenging realm. The null host signifies default + * credentials, which should be used when no {@link Credentials credentials} + * have been explictly supplied for the challenging host. Any previous + * credentials for the given realm on the given host will be overwritten. + * + * @param realm the authentication realm + * @param host the host the realm belongs to + * @param credentials the authentication {@link Credentials credentials} + * for the given realm. + * + * @see #getCredentials(String, String) + * @see #setProxyCredentials(String, String, Credentials) + * + * @deprecated use #setCredentials(AuthScope, Credentials) + */ + + public synchronized void setCredentials(String realm, String host, Credentials credentials) { + LOG.trace("enter HttpState.setCredentials(String, String, Credentials)"); + credMap.put(new AuthScope(host, AuthScope.ANY_PORT, realm, AuthScope.ANY_SCHEME), credentials); + } + + /** + * Sets the {@link Credentials credentials} for the given authentication + * scope. Any previous credentials for the given scope will be overwritten. + * + * @param authscope the {@link AuthScope authentication scope} + * @param credentials the authentication {@link Credentials credentials} + * for the given scope. + * + * @see #getCredentials(AuthScope) + * @see #setProxyCredentials(AuthScope, Credentials) + * + * @since 3.0 + */ + public synchronized void setCredentials(final AuthScope authscope, final Credentials credentials) { + if (authscope == null) { + throw new IllegalArgumentException("Authentication scope may not be null"); + } + LOG.trace("enter HttpState.setCredentials(AuthScope, Credentials)"); + credMap.put(authscope, credentials); + } + + /** + * Find matching {@link Credentials credentials} for the given authentication scope. + * + * @param map the credentials hash map + * @param token the {@link AuthScope authentication scope} + * @return the credentials + * + */ + private static Credentials matchCredentials(final HashMap map, final AuthScope authscope) { + // see if we get a direct hit + Credentials creds = (Credentials)map.get(authscope); + if (creds == null) { + // Nope. + // Do a full scan + int bestMatchFactor = -1; + AuthScope bestMatch = null; + Iterator items = map.keySet().iterator(); + while (items.hasNext()) { + AuthScope current = (AuthScope)items.next(); + int factor = authscope.match(current); + if (factor > bestMatchFactor) { + bestMatchFactor = factor; + bestMatch = current; + } + } + if (bestMatch != null) { + creds = (Credentials)map.get(bestMatch); + } + } + return creds; + } + + /** + * Get the {@link Credentials credentials} for the given authentication scope on the + * given host. + * + * If the realm exists on host, return the coresponding credentials. + * If the host exists with a null realm, return the corresponding + * credentials. + * If the realm exists with a null host, return the + * corresponding credentials. If the realm does not exist, return + * the default Credentials. If there are no default credentials, return + * null. + * + * @param realm the authentication realm + * @param host the host the realm is on + * @return the credentials + * + * @see #setCredentials(String, String, Credentials) + * + * @deprecated use #getCredentials(AuthScope) + */ + + public synchronized Credentials getCredentials(String realm, String host) { + LOG.trace("enter HttpState.getCredentials(String, String"); + return matchCredentials(this.credMap, + new AuthScope(host, AuthScope.ANY_PORT, realm, AuthScope.ANY_SCHEME)); + } + + /** + * Get the {@link Credentials credentials} for the given authentication scope. + * + * @param authscope the {@link AuthScope authentication scope} + * @return the credentials + * + * @see #setCredentials(AuthScope, Credentials) + * + * @since 3.0 + */ + public synchronized Credentials getCredentials(final AuthScope authscope) { + if (authscope == null) { + throw new IllegalArgumentException("Authentication scope may not be null"); + } + LOG.trace("enter HttpState.getCredentials(AuthScope)"); + return matchCredentials(this.credMap, authscope); + } + + /** + * Sets the {@link Credentials credentials} for the given proxy authentication + * realm on the given proxy host. The null proxy realm signifies + * default credentials for the given proxy host, which should be used when no + * {@link Credentials credentials} have been explictly supplied for the + * challenging proxy realm. The null proxy host signifies default + * credentials, which should be used when no {@link Credentials credentials} + * have been explictly supplied for the challenging proxy host. Any previous + * credentials for the given proxy realm on the given proxy host will be + * overwritten. + * + * @param realm the authentication realm + * @param proxyHost the proxy host + * @param credentials the authentication credentials for the given realm + * + * @see #getProxyCredentials(AuthScope) + * @see #setCredentials(AuthScope, Credentials) + * + * @deprecated use #setProxyCredentials(AuthScope, Credentials) + */ + public synchronized void setProxyCredentials( + String realm, + String proxyHost, + Credentials credentials + ) { + LOG.trace("enter HttpState.setProxyCredentials(String, String, Credentials"); + proxyCred.put(new AuthScope(proxyHost, AuthScope.ANY_PORT, realm, AuthScope.ANY_SCHEME), credentials); + } + + /** + * Sets the {@link Credentials proxy credentials} for the given authentication + * realm. Any previous credentials for the given realm will be overwritten. + * + * @param authscope the {@link AuthScope authentication scope} + * @param credentials the authentication {@link Credentials credentials} + * for the given realm. + * + * @see #getProxyCredentials(AuthScope) + * @see #setCredentials(AuthScope, Credentials) + * + * @since 3.0 + */ + public synchronized void setProxyCredentials(final AuthScope authscope, + final Credentials credentials) + { + if (authscope == null) { + throw new IllegalArgumentException("Authentication scope may not be null"); + } + LOG.trace("enter HttpState.setProxyCredentials(AuthScope, Credentials)"); + proxyCred.put(authscope, credentials); + } + + /** + * Get the {@link Credentials credentials} for the proxy host with the given + * authentication scope. + * + * If the realm exists on host, return the coresponding credentials. + * If the host exists with a null realm, return the corresponding + * credentials. + * If the realm exists with a null host, return the + * corresponding credentials. If the realm does not exist, return + * the default Credentials. If there are no default credentials, return + * null. + * + * @param realm the authentication realm + * @param proxyHost the proxy host the realm is on + * @return the credentials + * @see #setProxyCredentials(String, String, Credentials) + * + * @deprecated use #getProxyCredentials(AuthScope) + */ + public synchronized Credentials getProxyCredentials(String realm, String proxyHost) { + LOG.trace("enter HttpState.getCredentials(String, String"); + return matchCredentials(this.proxyCred, + new AuthScope(proxyHost, AuthScope.ANY_PORT, realm, AuthScope.ANY_SCHEME)); + } + + /** + * Get the {@link Credentials proxy credentials} for the given authentication scope. + * + * @param authscope the {@link AuthScope authentication scope} + * @return the credentials + * + * @see #setProxyCredentials(AuthScope, Credentials) + * + * @since 3.0 + */ + public synchronized Credentials getProxyCredentials(final AuthScope authscope) { + if (authscope == null) { + throw new IllegalArgumentException("Authentication scope may not be null"); + } + LOG.trace("enter HttpState.getProxyCredentials(AuthScope)"); + return matchCredentials(this.proxyCred, authscope); + } + + /** + * Returns a string representation of this HTTP state. + * + * @return The string representation of the HTTP state. + * + * @see java.lang.Object#toString() + */ + public synchronized String toString() { + StringBuffer sbResult = new StringBuffer(); + + sbResult.append("["); + sbResult.append(getCredentialsStringRepresentation(proxyCred)); + sbResult.append(" | "); + sbResult.append(getCredentialsStringRepresentation(credMap)); + sbResult.append(" | "); + sbResult.append(getCookiesStringRepresentation(cookiesMap.values())); // <- IA/HERITRIX CHANGE + sbResult.append("]"); + + String strResult = sbResult.toString(); + + return strResult; + } + + /** + * Returns a string representation of the credentials. + * @param credMap The credentials. + * @return The string representation. + */ + private static String getCredentialsStringRepresentation(final Map credMap) { + StringBuffer sbResult = new StringBuffer(); + Iterator iter = credMap.keySet().iterator(); + while (iter.hasNext()) { + Object key = iter.next(); + Credentials cred = (Credentials) credMap.get(key); + if (sbResult.length() > 0) { + sbResult.append(", "); + } + sbResult.append(key); + sbResult.append("#"); + sbResult.append(cred.toString()); + } + return sbResult.toString(); + } + + /** + * Returns a string representation of the cookies. + * @param cookies The cookies + * @return The string representation. + */ + private static String getCookiesStringRepresentation(final Collection cookies) { // <- IA/HERITRIX CHANGE + StringBuffer sbResult = new StringBuffer(); + Iterator iter = cookies.iterator(); + while (iter.hasNext()) { + Cookie ck = (Cookie) iter.next(); + if (sbResult.length() > 0) { + sbResult.append("#"); + } + sbResult.append(ck.toExternalForm()); + } + return sbResult.toString(); + } + + /** + * Clears all credentials. + */ + public void clearCredentials() { + this.credMap.clear(); + } + + /** + * Clears all proxy credentials. + */ + public void clearProxyCredentials() { + this.proxyCred.clear(); + } + + /** + * Clears all cookies. + */ + public void clearCookies() { +// BEGIN IA/HERITRIX CHANGES +// this.cookiesArrayList.clear(); + this.cookiesMap.clear(); +// END IA/HERITRIX CHANGES + } + + /** + * Clears the state information (all cookies, credentials and proxy credentials). + */ + public void clear() { + clearCookies(); + clearCredentials(); + clearProxyCredentials(); + } +} diff --git a/commons/src/main/java/org/apache/commons/httpclient/cookie/CookieSpec.java b/commons/src/main/java/org/apache/commons/httpclient/cookie/CookieSpec.java new file mode 100644 index 00000000..5b14bff4 --- /dev/null +++ b/commons/src/main/java/org/apache/commons/httpclient/cookie/CookieSpec.java @@ -0,0 +1,264 @@ +/* + * $Header: /home/jerenkrantz/tmp/commons/commons-convert/cvs/home/cvs/jakarta-commons//httpclient/src/java/org/apache/commons/httpclient/cookie/CookieSpec.java,v 1.11 2004/09/14 20:11:31 olegk Exp $ + * $Revision$ + * $Date$ + * + * ==================================================================== + * + * Copyright 2002-2004 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ + +package org.apache.commons.httpclient.cookie; + +import java.util.Collection; +import java.util.SortedMap; // <- IA/HERITRIX CHANGE + +import org.apache.commons.httpclient.Header; +import org.apache.commons.httpclient.NameValuePair; +import org.apache.commons.httpclient.Cookie; + +/** + * Defines the cookie management specification. + *

Cookie management specification must define + *

    + *
  • rules of parsing "Set-Cookie" header + *
  • rules of validation of parsed cookies + *
  • formatting of "Cookie" header + *
+ * for a given host, port and path of origin + * + * @author Oleg Kalnichevski + * @author Jeff Dever + * + * @since 2.0 + */ +@SuppressWarnings("unchecked") // <- IA/HERITRIX CHANGE +public interface CookieSpec { + + /** Path delimiter */ + static final String PATH_DELIM = "/"; + + /** Path delimiting charachter */ + static final char PATH_DELIM_CHAR = PATH_DELIM.charAt(0); + + /** + * Parse the "Set-Cookie" header value into Cookie array. + * + *

This method will not perform the validation of the resultant + * {@link Cookie}s

+ * + * @see #validate(String, int, String, boolean, Cookie) + * + * @param host the host which sent the Set-Cookie header + * @param port the port which sent the Set-Cookie header + * @param path the path which sent the Set-Cookie header + * @param secure true when the Set-Cookie header + * was received over secure conection + * @param header the Set-Cookie received from the server + * @return an array of Cookies parsed from the Set-Cookie value + * @throws MalformedCookieException if an exception occurs during parsing + * @throws IllegalArgumentException if an input parameter is illegal + */ + Cookie[] parse(String host, int port, String path, boolean secure, + final String header) + throws MalformedCookieException, IllegalArgumentException; + + /** + * Parse the "Set-Cookie" Header into an array of Cookies. + * + *

This method will not perform the validation of the resultant + * {@link Cookie}s

+ * + * @see #validate(String, int, String, boolean, Cookie) + * + * @param host the host which sent the Set-Cookie header + * @param port the port which sent the Set-Cookie header + * @param path the path which sent the Set-Cookie header + * @param secure true when the Set-Cookie header + * was received over secure conection + * @param header the Set-Cookie received from the server + * @return an array of Cookies parsed from the header + * @throws MalformedCookieException if an exception occurs during parsing + * @throws IllegalArgumentException if an input parameter is illegal + */ + Cookie[] parse(String host, int port, String path, boolean secure, + final Header header) + throws MalformedCookieException, IllegalArgumentException; + + /** + * Parse the cookie attribute and update the corresponsing Cookie + * properties. + * + * @param attribute cookie attribute from the Set-Cookie + * @param cookie the to be updated + * @throws MalformedCookieException if an exception occurs during parsing + * @throws IllegalArgumentException if an input parameter is illegal + */ + void parseAttribute(NameValuePair attribute, Cookie cookie) + throws MalformedCookieException, IllegalArgumentException; + + /** + * Validate the cookie according to validation rules defined by the + * cookie specification. + * + * @param host the host from which the {@link Cookie} was received + * @param port the port from which the {@link Cookie} was received + * @param path the path from which the {@link Cookie} was received + * @param secure true when the {@link Cookie} was received + * using a secure connection + * @param cookie the Cookie to validate + * @throws MalformedCookieException if the cookie is invalid + * @throws IllegalArgumentException if an input parameter is illegal + */ + void validate(String host, int port, String path, boolean secure, + final Cookie cookie) + throws MalformedCookieException, IllegalArgumentException; + + + /** + * Sets the {@link Collection} of date patterns used for parsing. The String patterns must be + * compatible with {@link java.text.SimpleDateFormat}. + * + * @param datepatterns collection of date patterns + */ + void setValidDateFormats(Collection datepatterns); + + /** + * Returns the {@link Collection} of date patterns used for parsing. The String patterns are compatible + * with the {@link java.text.SimpleDateFormat}. + * + * @return collection of date patterns + */ + Collection getValidDateFormats(); + + /** + * Determines if a Cookie matches a location. + * + * @param host the host to which the request is being submitted + * @param port the port to which the request is being submitted + * @param path the path to which the request is being submitted + * @param secure true if the request is using a secure connection + * @param cookie the Cookie to be matched + * + * @return true if the cookie should be submitted with a request + * with given attributes, false otherwise. + */ + boolean match(String host, int port, String path, boolean secure, + final Cookie cookie); + + /** + * Determines which of an array of Cookies matches a location. + * + * @param host the host to which the request is being submitted + * @param port the port to which the request is being submitted + * (currenlty ignored) + * @param path the path to which the request is being submitted + * @param secure true if the request is using a secure protocol + * @param cookies an array of Cookies to be matched + * + * @return true if the cookie should be submitted with a request + * with given attributes, false otherwise. + * +// BEGIN IA/HERITRIX CHANGES + * @deprecated use match(String, int, String, boolean, SortedMap) +// END IA/HERITRIX CHANGES + */ + Cookie[] match(String host, int port, String path, boolean secure, + final Cookie cookies[]); + +// BEGIN IA/HERITRIX CHANGES + /** + * Determines which of an array of Cookies matches a location. + * + * If the SortedMap comes from an HttpState and is not itself + * thread-safe, it may be necessary to synchronize on the HttpState + * instance to protect against concurrent modification. + * + * @param host the host to which the request is being submitted + * @param port the port to which the request is being submitted + * (currenlty ignored) + * @param path the path to which the request is being submitted + * @param secure true if the request is using a secure protocol + * @param cookies SortedMap of Cookies to be matched + * + * @return true if the cookie should be submitted with a request + * with given attributes, false otherwise. + */ + Cookie[] match(String domain, int port, String path, boolean secure, SortedMap cookiesMap); +// END IA/HERITRIX CHANGES + + /** + * Performs domain-match as defined by the cookie specification. + * @param host The target host. + * @param domain The cookie domain attribute. + * @return true if the specified host matches the given domain. + * + * @since 3.0 + */ + boolean domainMatch(String host, String domain); + + /** + * Performs path-match as defined by the cookie specification. + * @param path The target path. + * @param topmostPath The cookie path attribute. + * @return true if the paths match + * + * @since 3.0 + */ + boolean pathMatch(String path, String topmostPath); + + /** + * Create a "Cookie" header value for an array of cookies. + * + * @param cookie the cookie to be formatted as string + * @return a string suitable for sending in a "Cookie" header. + */ + String formatCookie(Cookie cookie); + + /** + * Create a "Cookie" header value for an array of cookies. + * + * @param cookies the Cookies to be formatted + * @return a string suitable for sending in a Cookie header. + * @throws IllegalArgumentException if an input parameter is illegal + */ + String formatCookies(Cookie[] cookies) throws IllegalArgumentException; + + /** + * Create a "Cookie" Header for an array of Cookies. + * + * @param cookies the Cookies format into a Cookie header + * @return a Header for the given Cookies. + * @throws IllegalArgumentException if an input parameter is illegal + */ + Header formatCookieHeader(Cookie[] cookies) throws IllegalArgumentException; + + /** + * Create a "Cookie" Header for single Cookie. + * + * @param cookie the Cookie format as a Cookie header + * @return a Cookie header. + * @throws IllegalArgumentException if an input parameter is illegal + */ + Header formatCookieHeader(Cookie cookie) throws IllegalArgumentException; + +} diff --git a/commons/src/main/java/org/apache/commons/httpclient/cookie/CookieSpecBase.java b/commons/src/main/java/org/apache/commons/httpclient/cookie/CookieSpecBase.java new file mode 100644 index 00000000..d15121d6 --- /dev/null +++ b/commons/src/main/java/org/apache/commons/httpclient/cookie/CookieSpecBase.java @@ -0,0 +1,716 @@ +/* + * $Header: /home/jerenkrantz/tmp/commons/commons-convert/cvs/home/cvs/jakarta-commons//httpclient/src/java/org/apache/commons/httpclient/cookie/CookieSpecBase.java,v 1.28 2004/11/06 19:15:42 mbecke Exp $ + * $Revision$ + * $Date$ + * + * ==================================================================== + * + * Copyright 2002-2004 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ + +package org.apache.commons.httpclient.cookie; + +import java.util.Collection; +import java.util.Date; +import java.util.Iterator; // <- IA/HERITRIX CHANGE +import java.util.LinkedList; +import java.util.List; +import java.util.SortedMap; // <- IA/HERITRIX CHANGE + +import org.apache.commons.httpclient.Cookie; +import org.apache.commons.httpclient.Header; +import org.apache.commons.httpclient.HeaderElement; +import org.apache.commons.httpclient.NameValuePair; +import org.apache.commons.httpclient.util.DateParseException; +import org.apache.commons.httpclient.util.DateUtil; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import com.sleepycat.collections.StoredIterator; // <- IA/HERITRIX CHANGE + +/** + * + * Cookie management functions shared by all specification. + * + * @author B.C. Holmes + * @author Park, Sung-Gu + * @author Doug Sale + * @author Rod Waldhoff + * @author dIon Gillard + * @author Sean C. Sullivan + * @author John Evans + * @author Marc A. Saegesser + * @author Oleg Kalnichevski + * @author Mike Bowler + * + * @since 2.0 + */ +@SuppressWarnings("unchecked") // <- IA/HERITRIX CHANGE +public class CookieSpecBase implements CookieSpec { + + /** Log object */ + protected static final Log LOG = LogFactory.getLog(CookieSpec.class); + + /** Valid date patterns */ + private Collection datepatterns = null; + + /** Default constructor */ + public CookieSpecBase() { + super(); + } + + + /** + * Parses the Set-Cookie value into an array of Cookies. + * + *

The syntax for the Set-Cookie response header is: + * + * 

+      * set-cookie      =    "Set-Cookie:" cookies
+      * cookies         =    1#cookie
+      * cookie          =    NAME "=" VALUE * (";" cookie-av)
+      * NAME            =    attr
+      * VALUE           =    value
+      * cookie-av       =    "Comment" "=" value
+      *                 |    "Domain" "=" value
+      *                 |    "Max-Age" "=" value
+      *                 |    "Path" "=" value
+      *                 |    "Secure"
+      *                 |    "Version" "=" 1*DIGIT
+      *
+ * + * @param host the host from which the Set-Cookie value was + * received + * @param port the port from which the Set-Cookie value was + * received + * @param path the path from which the Set-Cookie value was + * received + * @param secure true when the Set-Cookie value was + * received over secure conection + * @param header the Set-Cookie received from the server + * @return an array of Cookies parsed from the Set-Cookie value + * @throws MalformedCookieException if an exception occurs during parsing + */ + public Cookie[] parse(String host, int port, String path, + boolean secure, final String header) + throws MalformedCookieException { + + LOG.trace("enter CookieSpecBase.parse(" + + "String, port, path, boolean, Header)"); + + if (host == null) { + throw new IllegalArgumentException( + "Host of origin may not be null"); + } + if (host.trim().equals("")) { + throw new IllegalArgumentException( + "Host of origin may not be blank"); + } + if (port < 0) { + throw new IllegalArgumentException("Invalid port: " + port); + } + if (path == null) { + throw new IllegalArgumentException( + "Path of origin may not be null."); + } + if (header == null) { + throw new IllegalArgumentException("Header may not be null."); + } + + if (path.trim().equals("")) { + path = PATH_DELIM; + } + host = host.toLowerCase(); + + String defaultPath = path; + int lastSlashIndex = defaultPath.lastIndexOf(PATH_DELIM); + if (lastSlashIndex >= 0) { + if (lastSlashIndex == 0) { + //Do not remove the very first slash + lastSlashIndex = 1; + } + defaultPath = defaultPath.substring(0, lastSlashIndex); + } + + HeaderElement[] headerElements = null; + + boolean isNetscapeCookie = false; + int i1 = header.toLowerCase().indexOf("expires="); + if (i1 != -1) { + i1 += "expires=".length(); + int i2 = header.indexOf(";", i1); + if (i2 == -1) { + i2 = header.length(); + } + try { + DateUtil.parseDate(header.substring(i1, i2), this.datepatterns); + isNetscapeCookie = true; + } catch (DateParseException e) { + // Does not look like a valid expiry date + } + } + if (isNetscapeCookie) { + headerElements = new HeaderElement[] { + new HeaderElement(header.toCharArray()) + }; + } else { + headerElements = HeaderElement.parseElements(header.toCharArray()); + } + + Cookie[] cookies = new Cookie[headerElements.length]; + + for (int i = 0; i < headerElements.length; i++) { + + HeaderElement headerelement = headerElements[i]; + Cookie cookie = null; + try { + cookie = new Cookie(host, + headerelement.getName(), + headerelement.getValue(), + defaultPath, + null, + false); + } catch (IllegalArgumentException e) { + throw new MalformedCookieException(e.getMessage()); + } + // cycle through the parameters + NameValuePair[] parameters = headerelement.getParameters(); + // could be null. In case only a header element and no parameters. + if (parameters != null) { + + for (int j = 0; j < parameters.length; j++) { + parseAttribute(parameters[j], cookie); + } + } + cookies[i] = cookie; + } + return cookies; + } + + + /** + * Parse the "Set-Cookie" {@link Header} into an array of {@link + * Cookie}s. + * + *

The syntax for the Set-Cookie response header is: + * + * 

+      * set-cookie      =    "Set-Cookie:" cookies
+      * cookies         =    1#cookie
+      * cookie          =    NAME "=" VALUE * (";" cookie-av)
+      * NAME            =    attr
+      * VALUE           =    value
+      * cookie-av       =    "Comment" "=" value
+      *                 |    "Domain" "=" value
+      *                 |    "Max-Age" "=" value
+      *                 |    "Path" "=" value
+      *                 |    "Secure"
+      *                 |    "Version" "=" 1*DIGIT
+      *
+ * + * @param host the host from which the Set-Cookie header was + * received + * @param port the port from which the Set-Cookie header was + * received + * @param path the path from which the Set-Cookie header was + * received + * @param secure true when the Set-Cookie header was + * received over secure conection + * @param header the Set-Cookie received from the server + * @return an array of Cookies parsed from the "Set-Cookie" + * header + * @throws MalformedCookieException if an exception occurs during parsing + */ + public Cookie[] parse( + String host, int port, String path, boolean secure, final Header header) + throws MalformedCookieException { + + LOG.trace("enter CookieSpecBase.parse(" + + "String, port, path, boolean, String)"); + if (header == null) { + throw new IllegalArgumentException("Header may not be null."); + } + return parse(host, port, path, secure, header.getValue()); + } + + + /** + * Parse the cookie attribute and update the corresponsing {@link Cookie} + * properties. + * + * @param attribute {@link HeaderElement} cookie attribute from the + * Set- Cookie + * @param cookie {@link Cookie} to be updated + * @throws MalformedCookieException if an exception occurs during parsing + */ + + public void parseAttribute( + final NameValuePair attribute, final Cookie cookie) + throws MalformedCookieException { + + if (attribute == null) { + throw new IllegalArgumentException("Attribute may not be null."); + } + if (cookie == null) { + throw new IllegalArgumentException("Cookie may not be null."); + } + final String paramName = attribute.getName().toLowerCase(); + String paramValue = attribute.getValue(); + + if (paramName.equals("path")) { + + if ((paramValue == null) || (paramValue.trim().equals(""))) { + paramValue = "/"; + } + cookie.setPath(paramValue); + cookie.setPathAttributeSpecified(true); + + } else if (paramName.equals("domain")) { + + if (paramValue == null) { + throw new MalformedCookieException( + "Missing value for domain attribute"); + } + if (paramValue.trim().equals("")) { + throw new MalformedCookieException( + "Blank value for domain attribute"); + } + cookie.setDomain(paramValue); + cookie.setDomainAttributeSpecified(true); + + } else if (paramName.equals("max-age")) { + + if (paramValue == null) { + throw new MalformedCookieException( + "Missing value for max-age attribute"); + } + int age; + try { + age = Integer.parseInt(paramValue); + } catch (NumberFormatException e) { + throw new MalformedCookieException ("Invalid max-age " + + "attribute: " + e.getMessage()); + } + cookie.setExpiryDate( + new Date(System.currentTimeMillis() + age * 1000L)); + + } else if (paramName.equals("secure")) { + + cookie.setSecure(true); + + } else if (paramName.equals("comment")) { + + cookie.setComment(paramValue); + + } else if (paramName.equals("expires")) { + + if (paramValue == null) { + throw new MalformedCookieException( + "Missing value for expires attribute"); + } + + try { + cookie.setExpiryDate(DateUtil.parseDate(paramValue, this.datepatterns)); + } catch (DateParseException dpe) { + LOG.debug("Error parsing cookie date", dpe); + throw new MalformedCookieException( + "Unable to parse expiration date parameter: " + + paramValue); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Unrecognized cookie attribute: " + + attribute.toString()); + } + } + } + + + public Collection getValidDateFormats() { + return this.datepatterns; + } + + public void setValidDateFormats(final Collection datepatterns) { + this.datepatterns = datepatterns; + } + + /** + * Performs most common {@link Cookie} validation + * + * @param host the host from which the {@link Cookie} was received + * @param port the port from which the {@link Cookie} was received + * @param path the path from which the {@link Cookie} was received + * @param secure true when the {@link Cookie} was received using a + * secure connection + * @param cookie The cookie to validate. + * @throws MalformedCookieException if an exception occurs during + * validation + */ + + public void validate(String host, int port, String path, + boolean secure, final Cookie cookie) + throws MalformedCookieException { + + LOG.trace("enter CookieSpecBase.validate(" + + "String, port, path, boolean, Cookie)"); + if (host == null) { + throw new IllegalArgumentException( + "Host of origin may not be null"); + } + if (host.trim().equals("")) { + throw new IllegalArgumentException( + "Host of origin may not be blank"); + } + if (port < 0) { + throw new IllegalArgumentException("Invalid port: " + port); + } + if (path == null) { + throw new IllegalArgumentException( + "Path of origin may not be null."); + } + if (path.trim().equals("")) { + path = PATH_DELIM; + } + host = host.toLowerCase(); + // check version + if (cookie.getVersion() < 0) { + throw new MalformedCookieException ("Illegal version number " + + cookie.getValue()); + } + + // security check... we musn't allow the server to give us an + // invalid domain scope + + // Validate the cookies domain attribute. NOTE: Domains without + // any dots are allowed to support hosts on private LANs that don't + // have DNS names. Since they have no dots, to domain-match the + // request-host and domain must be identical for the cookie to sent + // back to the origin-server. + if (host.indexOf(".") >= 0) { + // Not required to have at least two dots. RFC 2965. + // A Set-Cookie2 with Domain=ajax.com will be accepted. + + // domain must match host + if (!host.endsWith(cookie.getDomain())) { + String s = cookie.getDomain(); + if (s.startsWith(".")) { + s = s.substring(1, s.length()); + } + if (!host.equals(s)) { + throw new MalformedCookieException( + "Illegal domain attribute \"" + cookie.getDomain() + + "\". Domain of origin: \"" + host + "\""); + } + } + } else { + if (!host.equals(cookie.getDomain())) { + throw new MalformedCookieException( + "Illegal domain attribute \"" + cookie.getDomain() + + "\". Domain of origin: \"" + host + "\""); + } + } + + // another security check... we musn't allow the server to give us a + // cookie that doesn't match this path + + if (!path.startsWith(cookie.getPath())) { + throw new MalformedCookieException( + "Illegal path attribute \"" + cookie.getPath() + + "\". Path of origin: \"" + path + "\""); + } + } + + + /** + * Return true if the cookie should be submitted with a request + * with given attributes, false otherwise. + * @param host the host to which the request is being submitted + * @param port the port to which the request is being submitted (ignored) + * @param path the path to which the request is being submitted + * @param secure true if the request is using a secure connection + * @param cookie {@link Cookie} to be matched + * @return true if the cookie matches the criterium + */ + + public boolean match(String host, int port, String path, + boolean secure, final Cookie cookie) { + + LOG.trace("enter CookieSpecBase.match(" + + "String, int, String, boolean, Cookie"); + + if (host == null) { + throw new IllegalArgumentException( + "Host of origin may not be null"); + } + if (host.trim().equals("")) { + throw new IllegalArgumentException( + "Host of origin may not be blank"); + } + if (port < 0) { + throw new IllegalArgumentException("Invalid port: " + port); + } + if (path == null) { + throw new IllegalArgumentException( + "Path of origin may not be null."); + } + if (cookie == null) { + throw new IllegalArgumentException("Cookie may not be null"); + } + if (path.trim().equals("")) { + path = PATH_DELIM; + } + host = host.toLowerCase(); + if (cookie.getDomain() == null) { + LOG.warn("Invalid cookie state: domain not specified"); + return false; + } + if (cookie.getPath() == null) { + LOG.warn("Invalid cookie state: path not specified"); + return false; + } + + return + // only add the cookie if it hasn't yet expired + (cookie.getExpiryDate() == null + || cookie.getExpiryDate().after(new Date())) + // and the domain pattern matches + && (domainMatch(host, cookie.getDomain())) + // and the path is null or matching + && (pathMatch(path, cookie.getPath())) + // and if the secure flag is set, only if the request is + // actually secure + && (cookie.getSecure() ? secure : true); + } + + /** + * Performs domain-match as implemented in common browsers. + * @param host The target host. + * @param domain The cookie domain attribute. + * @return true if the specified host matches the given domain. + */ + public boolean domainMatch(final String host, String domain) { + if (host.equals(domain)) { + return true; + } + if (!domain.startsWith(".")) { + domain = "." + domain; + } + return host.endsWith(domain) || host.equals(domain.substring(1)); + } + + /** + * Performs path-match as implemented in common browsers. + * @param path The target path. + * @param topmostPath The cookie path attribute. + * @return true if the paths match + */ + public boolean pathMatch(final String path, final String topmostPath) { + boolean match = path.startsWith (topmostPath); + // if there is a match and these values are not exactly the same we have + // to make sure we're not matcing "/foobar" and "/foo" + if (match && path.length() != topmostPath.length()) { + if (!topmostPath.endsWith(PATH_DELIM)) { + match = (path.charAt(topmostPath.length()) == PATH_DELIM_CHAR); + } + } + return match; + } + + /** + * Return an array of {@link Cookie}s that should be submitted with a + * request with given attributes, false otherwise. + * @param host the host to which the request is being submitted + * @param port the port to which the request is being submitted (currently + * ignored) + * @param path the path to which the request is being submitted + * @param secure true if the request is using a secure protocol + * @param cookies an array of Cookies to be matched + * @return an array of Cookies matching the criterium + * +// BEGIN IA/HERITRIX CHANGES + * @deprecated use match(String, int, String, boolean, SortedMap) +// END IA/HERITRIX CHANGES + */ + + public Cookie[] match(String host, int port, String path, + boolean secure, final Cookie cookies[]) { + + LOG.trace("enter CookieSpecBase.match(" + + "String, int, String, boolean, Cookie[])"); + + if (cookies == null) { + return null; + } + List matching = new LinkedList(); + for (int i = 0; i < cookies.length; i++) { + if (match(host, port, path, secure, cookies[i])) { + addInPathOrder(matching, cookies[i]); + } + } + return (Cookie[]) matching.toArray(new Cookie[matching.size()]); + } + +// BEGIN IA/HERITRIX CHANGES + /** + * Return an array of {@link Cookie}s that should be submitted with a + * request with given attributes, false otherwise. + * + * If the SortedMap comes from an HttpState and is not itself + * thread-safe, it may be necessary to synchronize on the HttpState + * instance to protect against concurrent modification. + * + * @param host the host to which the request is being submitted + * @param port the port to which the request is being submitted (currently + * ignored) + * @param path the path to which the request is being submitted + * @param secure true if the request is using a secure protocol + * @param cookies SortedMap of Cookies to be matched + * @return an array of Cookies matching the criterium + */ + + public Cookie[] match(String host, int port, String path, + boolean secure, final SortedMap cookies) { + + LOG.trace("enter CookieSpecBase.match(" + + "String, int, String, boolean, SortedMap)"); + + // TODO: skip meaningless 'narrowing' when host is a numeric IP + // (harmless in the meantime) + + if (cookies == null) { + return null; + } + List matching = new LinkedList(); + String narrowHost = host; + do { + Iterator iter = cookies.subMap(narrowHost, + narrowHost + Cookie.DOMAIN_OVERBOUNDS).values().iterator(); + while (iter.hasNext()) { + Cookie cookie = (Cookie) (iter.next()); + if (match(host, port, path, secure, cookie)) { + addInPathOrder(matching, cookie); + } + } + StoredIterator.close(iter); + int trimTo = narrowHost.indexOf('.', 1); + narrowHost = (trimTo < 0) ? null : narrowHost.substring(trimTo+1); + } while (narrowHost != null); + + return (Cookie[]) matching.toArray(new Cookie[matching.size()]); + } +// END IA/HERITRIX CHANGES + + /** + * Adds the given cookie into the given list in descending path order. That + * is, more specific path to least specific paths. This may not be the + * fastest algorythm, but it'll work OK for the small number of cookies + * we're generally dealing with. + * + * @param list - the list to add the cookie to + * @param addCookie - the Cookie to add to list + */ + private static void addInPathOrder(List list, Cookie addCookie) { + int i = 0; + + for (i = 0; i < list.size(); i++) { + Cookie c = (Cookie) list.get(i); + if (addCookie.compare(addCookie, c) > 0) { + break; + } + } + list.add(i, addCookie); + } + + /** + * Return a string suitable for sending in a "Cookie" header + * @param cookie a {@link Cookie} to be formatted as string + * @return a string suitable for sending in a "Cookie" header. + */ + public String formatCookie(Cookie cookie) { + LOG.trace("enter CookieSpecBase.formatCookie(Cookie)"); + if (cookie == null) { + throw new IllegalArgumentException("Cookie may not be null"); + } + StringBuffer buf = new StringBuffer(); + buf.append(cookie.getName()); + buf.append("="); + String s = cookie.getValue(); + if (s != null) { + buf.append(s); + } + return buf.toString(); + } + + /** + * Create a "Cookie" header value containing all {@link Cookie}s in + * cookies suitable for sending in a "Cookie" header + * @param cookies an array of {@link Cookie}s to be formatted + * @return a string suitable for sending in a Cookie header. + * @throws IllegalArgumentException if an input parameter is illegal + */ + + public String formatCookies(Cookie[] cookies) + throws IllegalArgumentException { + LOG.trace("enter CookieSpecBase.formatCookies(Cookie[])"); + if (cookies == null) { + throw new IllegalArgumentException("Cookie array may not be null"); + } + if (cookies.length == 0) { + throw new IllegalArgumentException("Cookie array may not be empty"); + } + + StringBuffer buffer = new StringBuffer(); + for (int i = 0; i < cookies.length; i++) { + if (i > 0) { + buffer.append("; "); + } + buffer.append(formatCookie(cookies[i])); + } + return buffer.toString(); + } + + + /** + * Create a "Cookie" {@link Header} containing all {@link Cookie}s + * in cookies. + * @param cookies an array of {@link Cookie}s to be formatted as a " + * Cookie" header + * @return a "Cookie" {@link Header}. + */ + public Header formatCookieHeader(Cookie[] cookies) { + LOG.trace("enter CookieSpecBase.formatCookieHeader(Cookie[])"); + return new Header("Cookie", formatCookies(cookies)); + } + + + /** + * Create a "Cookie" {@link Header} containing the {@link Cookie}. + * @param cookie Cookies to be formatted as a Cookie + * header + * @return a Cookie header. + */ + public Header formatCookieHeader(Cookie cookie) { + LOG.trace("enter CookieSpecBase.formatCookieHeader(Cookie)"); + return new Header("Cookie", formatCookie(cookie)); + } + +} diff --git a/commons/src/main/java/org/apache/commons/httpclient/cookie/IgnoreCookiesSpec.java b/commons/src/main/java/org/apache/commons/httpclient/cookie/IgnoreCookiesSpec.java new file mode 100644 index 00000000..82e1a105 --- /dev/null +++ b/commons/src/main/java/org/apache/commons/httpclient/cookie/IgnoreCookiesSpec.java @@ -0,0 +1,160 @@ +/* + * $Header: /home/jerenkrantz/tmp/commons/commons-convert/cvs/home/cvs/jakarta-commons//httpclient/src/java/org/apache/commons/httpclient/cookie/IgnoreCookiesSpec.java,v 1.6 2004/09/14 20:11:31 olegk Exp $ + * $Revision$ + * $Date$ + * + * ==================================================================== + * + * Copyright 2002-2004 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ + +package org.apache.commons.httpclient.cookie; + +import java.util.Collection; +import java.util.SortedMap; // <- IA/HERITRIX CHANGE + +import org.apache.commons.httpclient.Cookie; +import org.apache.commons.httpclient.Header; +import org.apache.commons.httpclient.NameValuePair; + +/** + * A cookie spec that does nothing. Cookies are neither parsed, formatted nor matched. + * It can be used to effectively disable cookies altogether. + * + * @since 3.0 + */ +@SuppressWarnings("unchecked") // <- IA/HERITRIX CHANGE +public class IgnoreCookiesSpec implements CookieSpec { + + /** + * + */ + public IgnoreCookiesSpec() { + super(); + } + + /** + * Returns an empty {@link Cookie cookie} array. All parameters are ignored. + */ + public Cookie[] parse(String host, int port, String path, boolean secure, String header) + throws MalformedCookieException { + return new Cookie[0]; + } + + /** + * @return null + */ + public Collection getValidDateFormats() { + return null; + } + + /** + * Does nothing. + */ + public void setValidDateFormats(Collection datepatterns) { + } + + /** + * @return null + */ + public String formatCookie(Cookie cookie) { + return null; + } + + /** + * @return null + */ + public Header formatCookieHeader(Cookie cookie) throws IllegalArgumentException { + return null; + } + + /** + * @return null + */ + public Header formatCookieHeader(Cookie[] cookies) throws IllegalArgumentException { + return null; + } + + /** + * @return null + */ + public String formatCookies(Cookie[] cookies) throws IllegalArgumentException { + return null; + } + + /** + * @return false + */ + public boolean match(String host, int port, String path, boolean secure, Cookie cookie) { + return false; + } + + /** + * Returns an empty {@link Cookie cookie} array. All parameters are ignored. + */ + public Cookie[] match(String host, int port, String path, boolean secure, Cookie[] cookies) { + return new Cookie[0]; + } + + /** + * Returns an empty {@link Cookie cookie} array. All parameters are ignored. + */ + public Cookie[] parse(String host, int port, String path, boolean secure, Header header) + throws MalformedCookieException, IllegalArgumentException { + return new Cookie[0]; + } + + /** + * Does nothing. + */ + public void parseAttribute(NameValuePair attribute, Cookie cookie) + throws MalformedCookieException, IllegalArgumentException { + } + + /** + * Does nothing. + */ + public void validate(String host, int port, String path, boolean secure, Cookie cookie) + throws MalformedCookieException, IllegalArgumentException { + } + + /** + * @return false + */ + public boolean domainMatch(final String host, final String domain) { + return false; + } + + /** + * @return false + */ + public boolean pathMatch(final String path, final String topmostPath) { + return false; + } + +// BEGIN IA/HERITRIX ADDITION + public Cookie[] match(String domain, int port, String path, boolean secure, + SortedMap cookiesMap) { + return new Cookie[0]; + } +// END IA/HERITRIX CHANGE +} diff --git a/commons/src/main/java/org/apache/commons/pool/impl/FairGenericObjectPool.java b/commons/src/main/java/org/apache/commons/pool/impl/FairGenericObjectPool.java new file mode 100644 index 00000000..569295dd --- /dev/null +++ b/commons/src/main/java/org/apache/commons/pool/impl/FairGenericObjectPool.java @@ -0,0 +1,310 @@ +/* FairGenericObjectPool +* +* $Id$ +* +* Created on Apr 7, 2006 +* +* Copyright (C) 2006 Internet Archive. +*/ + +package org.apache.commons.pool.impl; + +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.NoSuchElementException; + +import org.apache.commons.pool.PoolableObjectFactory; +import org.apache.commons.pool.impl.GenericKeyedObjectPool.ObjectTimestampPair; + +/** + * Version of GenericObjectPool which is 'fair' with respect to the client + * threads using {@link #borrowObject borrowObject}. Those which enter + * first will receive objects from the pool first. + * + * + * @see GenericObjectPool + * @author Gordon Mohr + * @version $Revision$ $Date$ + */ +@SuppressWarnings("unchecked") +public class FairGenericObjectPool extends GenericObjectPool { + + //--- constructors ----------------------------------------------- + // (all copied from superclass; only last adds one additional line of + // initialization and call to superclass) + + /** + * Create a new FairGenericObjectPool. + */ + public FairGenericObjectPool() { + this(null,DEFAULT_MAX_ACTIVE,DEFAULT_WHEN_EXHAUSTED_ACTION,DEFAULT_MAX_WAIT,DEFAULT_MAX_IDLE,DEFAULT_MIN_IDLE,DEFAULT_TEST_ON_BORROW,DEFAULT_TEST_ON_RETURN,DEFAULT_TIME_BETWEEN_EVICTION_RUNS_MILLIS,DEFAULT_NUM_TESTS_PER_EVICTION_RUN,DEFAULT_MIN_EVICTABLE_IDLE_TIME_MILLIS,DEFAULT_TEST_WHILE_IDLE); + } + + /** + * Create a new FairGenericObjectPool using the specified values. + * @param factory the (possibly null)PoolableObjectFactory to use to create, validate and destroy objects + */ + public FairGenericObjectPool(PoolableObjectFactory factory) { + this(factory,DEFAULT_MAX_ACTIVE,DEFAULT_WHEN_EXHAUSTED_ACTION,DEFAULT_MAX_WAIT,DEFAULT_MAX_IDLE,DEFAULT_MIN_IDLE,DEFAULT_TEST_ON_BORROW,DEFAULT_TEST_ON_RETURN,DEFAULT_TIME_BETWEEN_EVICTION_RUNS_MILLIS,DEFAULT_NUM_TESTS_PER_EVICTION_RUN,DEFAULT_MIN_EVICTABLE_IDLE_TIME_MILLIS,DEFAULT_TEST_WHILE_IDLE); + } + + /** + * Create a new FairGenericObjectPool using the specified values. + * @param factory the (possibly null)PoolableObjectFactory to use to create, validate and destroy objects + * @param config a non-null {@link GenericObjectPool.Config} describing my configuration + */ + public FairGenericObjectPool(PoolableObjectFactory factory, GenericObjectPool.Config config) { + this(factory,config.maxActive,config.whenExhaustedAction,config.maxWait,config.maxIdle,config.minIdle,config.testOnBorrow,config.testOnReturn,config.timeBetweenEvictionRunsMillis,config.numTestsPerEvictionRun,config.minEvictableIdleTimeMillis,config.testWhileIdle); + } + + /** + * Create a new FairGenericObjectPool using the specified values. + * @param factory the (possibly null)PoolableObjectFactory to use to create, validate and destroy objects + * @param maxActive the maximum number of objects that can be borrowed from me at one time (see {@link #setMaxActive}) + */ + public FairGenericObjectPool(PoolableObjectFactory factory, int maxActive) { + this(factory,maxActive,DEFAULT_WHEN_EXHAUSTED_ACTION,DEFAULT_MAX_WAIT,DEFAULT_MAX_IDLE,DEFAULT_MIN_IDLE,DEFAULT_TEST_ON_BORROW,DEFAULT_TEST_ON_RETURN,DEFAULT_TIME_BETWEEN_EVICTION_RUNS_MILLIS,DEFAULT_NUM_TESTS_PER_EVICTION_RUN,DEFAULT_MIN_EVICTABLE_IDLE_TIME_MILLIS,DEFAULT_TEST_WHILE_IDLE); + } + + /** + * Create a new FairGenericObjectPool using the specified values. + * @param factory the (possibly null)PoolableObjectFactory to use to create, validate and destroy objects + * @param maxActive the maximum number of objects that can be borrowed from me at one time (see {@link #setMaxActive}) + * @param whenExhaustedAction the action to take when the pool is exhausted (see {@link #getWhenExhaustedAction}) + * @param maxWait the maximum amount of time to wait for an idle object when the pool is exhausted an and whenExhaustedAction is {@link #WHEN_EXHAUSTED_BLOCK} (otherwise ignored) (see {@link #getMaxWait}) + */ + public FairGenericObjectPool(PoolableObjectFactory factory, int maxActive, byte whenExhaustedAction, long maxWait) { + this(factory,maxActive,whenExhaustedAction,maxWait,DEFAULT_MAX_IDLE,DEFAULT_MIN_IDLE,DEFAULT_TEST_ON_BORROW,DEFAULT_TEST_ON_RETURN,DEFAULT_TIME_BETWEEN_EVICTION_RUNS_MILLIS,DEFAULT_NUM_TESTS_PER_EVICTION_RUN,DEFAULT_MIN_EVICTABLE_IDLE_TIME_MILLIS,DEFAULT_TEST_WHILE_IDLE); + } + + /** + * Create a new FairGenericObjectPool using the specified values. + * @param factory the (possibly null)PoolableObjectFactory to use to create, validate and destroy objects + * @param maxActive the maximum number of objects that can be borrowed from me at one time (see {@link #setMaxActive}) + * @param whenExhaustedAction the action to take when the pool is exhausted (see {@link #getWhenExhaustedAction}) + * @param maxWait the maximum amount of time to wait for an idle object when the pool is exhausted an and whenExhaustedAction is {@link #WHEN_EXHAUSTED_BLOCK} (otherwise ignored) (see {@link #getMaxWait}) + * @param testOnBorrow whether or not to validate objects before they are returned by the {@link #borrowObject} method (see {@link #getTestOnBorrow}) + * @param testOnReturn whether or not to validate objects after they are returned to the {@link #returnObject} method (see {@link #getTestOnReturn}) + */ + public FairGenericObjectPool(PoolableObjectFactory factory, int maxActive, byte whenExhaustedAction, long maxWait, boolean testOnBorrow, boolean testOnReturn) { + this(factory,maxActive,whenExhaustedAction,maxWait,DEFAULT_MAX_IDLE,DEFAULT_MIN_IDLE,testOnBorrow,testOnReturn,DEFAULT_TIME_BETWEEN_EVICTION_RUNS_MILLIS,DEFAULT_NUM_TESTS_PER_EVICTION_RUN,DEFAULT_MIN_EVICTABLE_IDLE_TIME_MILLIS,DEFAULT_TEST_WHILE_IDLE); + } + + /** + * Create a new FairGenericObjectPool using the specified values. + * @param factory the (possibly null)PoolableObjectFactory to use to create, validate and destroy objects + * @param maxActive the maximum number of objects that can be borrowed from me at one time (see {@link #setMaxActive}) + * @param whenExhaustedAction the action to take when the pool is exhausted (see {@link #getWhenExhaustedAction}) + * @param maxWait the maximum amount of time to wait for an idle object when the pool is exhausted an and whenExhaustedAction is {@link #WHEN_EXHAUSTED_BLOCK} (otherwise ignored) (see {@link #getMaxWait}) + * @param maxIdle the maximum number of idle objects in my pool (see {@link #getMaxIdle}) + */ + public FairGenericObjectPool(PoolableObjectFactory factory, int maxActive, byte whenExhaustedAction, long maxWait, int maxIdle) { + this(factory,maxActive,whenExhaustedAction,maxWait,maxIdle,DEFAULT_MIN_IDLE,DEFAULT_TEST_ON_BORROW,DEFAULT_TEST_ON_RETURN,DEFAULT_TIME_BETWEEN_EVICTION_RUNS_MILLIS,DEFAULT_NUM_TESTS_PER_EVICTION_RUN,DEFAULT_MIN_EVICTABLE_IDLE_TIME_MILLIS,DEFAULT_TEST_WHILE_IDLE); + } + + /** + * Create a new FairGenericObjectPool using the specified values. + * @param factory the (possibly null)PoolableObjectFactory to use to create, validate and destroy objects + * @param maxActive the maximum number of objects that can be borrowed from me at one time (see {@link #setMaxActive}) + * @param whenExhaustedAction the action to take when the pool is exhausted (see {@link #getWhenExhaustedAction}) + * @param maxWait the maximum amount of time to wait for an idle object when the pool is exhausted an and whenExhaustedAction is {@link #WHEN_EXHAUSTED_BLOCK} (otherwise ignored) (see {@link #getMaxWait}) + * @param maxIdle the maximum number of idle objects in my pool (see {@link #getMaxIdle}) + * @param testOnBorrow whether or not to validate objects before they are returned by the {@link #borrowObject} method (see {@link #getTestOnBorrow}) + * @param testOnReturn whether or not to validate objects after they are returned to the {@link #returnObject} method (see {@link #getTestOnReturn}) + */ + public FairGenericObjectPool(PoolableObjectFactory factory, int maxActive, byte whenExhaustedAction, long maxWait, int maxIdle, boolean testOnBorrow, boolean testOnReturn) { + this(factory,maxActive,whenExhaustedAction,maxWait,maxIdle,DEFAULT_MIN_IDLE,testOnBorrow,testOnReturn,DEFAULT_TIME_BETWEEN_EVICTION_RUNS_MILLIS,DEFAULT_NUM_TESTS_PER_EVICTION_RUN,DEFAULT_MIN_EVICTABLE_IDLE_TIME_MILLIS,DEFAULT_TEST_WHILE_IDLE); + } + + /** + * Create a new FairGenericObjectPool using the specified values. + * @param factory the (possibly null)PoolableObjectFactory to use to create, validate and destroy objects + * @param maxActive the maximum number of objects that can be borrowed from me at one time (see {@link #setMaxActive}) + * @param whenExhaustedAction the action to take when the pool is exhausted (see {@link #setWhenExhaustedAction}) + * @param maxWait the maximum amount of time to wait for an idle object when the pool is exhausted an and whenExhaustedAction is {@link #WHEN_EXHAUSTED_BLOCK} (otherwise ignored) (see {@link #setMaxWait}) + * @param maxIdle the maximum number of idle objects in my pool (see {@link #setMaxIdle}) + * @param testOnBorrow whether or not to validate objects before they are returned by the {@link #borrowObject} method (see {@link #setTestOnBorrow}) + * @param testOnReturn whether or not to validate objects after they are returned to the {@link #returnObject} method (see {@link #setTestOnReturn}) + * @param timeBetweenEvictionRunsMillis the amount of time (in milliseconds) to sleep between examining idle objects for eviction (see {@link #setTimeBetweenEvictionRunsMillis}) + * @param numTestsPerEvictionRun the number of idle objects to examine per run within the idle object eviction thread (if any) (see {@link #setNumTestsPerEvictionRun}) + * @param minEvictableIdleTimeMillis the minimum number of milliseconds an object can sit idle in the pool before it is eligable for evcition (see {@link #setMinEvictableIdleTimeMillis}) + * @param testWhileIdle whether or not to validate objects in the idle object eviction thread, if any (see {@link #setTestWhileIdle}) + */ + public FairGenericObjectPool(PoolableObjectFactory factory, int maxActive, byte whenExhaustedAction, long maxWait, int maxIdle, boolean testOnBorrow, boolean testOnReturn, long timeBetweenEvictionRunsMillis, int numTestsPerEvictionRun, long minEvictableIdleTimeMillis, boolean testWhileIdle) { + this(factory, maxActive, whenExhaustedAction, maxWait, maxIdle, DEFAULT_MIN_IDLE, testOnBorrow, testOnReturn, timeBetweenEvictionRunsMillis, numTestsPerEvictionRun, minEvictableIdleTimeMillis, testWhileIdle); + } + + /** + * Create a new FairGenericObjectPool using the specified values. + * @param factory the (possibly null)PoolableObjectFactory to use to create, validate and destroy objects + * @param maxActive the maximum number of objects that can be borrowed from me at one time (see {@link #setMaxActive}) + * @param whenExhaustedAction the action to take when the pool is exhausted (see {@link #setWhenExhaustedAction}) + * @param maxWait the maximum amount of time to wait for an idle object when the pool is exhausted an and whenExhaustedAction is {@link #WHEN_EXHAUSTED_BLOCK} (otherwise ignored) (see {@link #setMaxWait}) + * @param maxIdle the maximum number of idle objects in my pool (see {@link #setMaxIdle}) + * @param minIdle the minimum number of idle objects in my pool (see {@link #setMinIdle}) + * @param testOnBorrow whether or not to validate objects before they are returned by the {@link #borrowObject} method (see {@link #setTestOnBorrow}) + * @param testOnReturn whether or not to validate objects after they are returned to the {@link #returnObject} method (see {@link #setTestOnReturn}) + * @param timeBetweenEvictionRunsMillis the amount of time (in milliseconds) to sleep between examining idle objects for eviction (see {@link #setTimeBetweenEvictionRunsMillis}) + * @param numTestsPerEvictionRun the number of idle objects to examine per run within the idle object eviction thread (if any) (see {@link #setNumTestsPerEvictionRun}) + * @param minEvictableIdleTimeMillis the minimum number of milliseconds an object can sit idle in the pool before it is eligable for evcition (see {@link #setMinEvictableIdleTimeMillis}) + * @param testWhileIdle whether or not to validate objects in the idle object eviction thread, if any (see {@link #setTestWhileIdle}) + */ + public FairGenericObjectPool(PoolableObjectFactory factory, int maxActive, byte whenExhaustedAction, long maxWait, int maxIdle, int minIdle, boolean testOnBorrow, boolean testOnReturn, long timeBetweenEvictionRunsMillis, int numTestsPerEvictionRun, long minEvictableIdleTimeMillis, boolean testWhileIdle) { + this(factory, maxActive, whenExhaustedAction, maxWait, maxIdle, minIdle, testOnBorrow, testOnReturn, timeBetweenEvictionRunsMillis, numTestsPerEvictionRun, minEvictableIdleTimeMillis, testWhileIdle, DEFAULT_SOFT_MIN_EVICTABLE_IDLE_TIME_MILLIS); + } + + /** + * Create a new FairGenericObjectPool using the specified values. + * @param factory the (possibly null)PoolableObjectFactory to use to create, validate and destroy objects + * @param maxActive the maximum number of objects that can be borrowed from me at one time (see {@link #setMaxActive}) + * @param whenExhaustedAction the action to take when the pool is exhausted (see {@link #setWhenExhaustedAction}) + * @param maxWait the maximum amount of time to wait for an idle object when the pool is exhausted an and whenExhaustedAction is {@link #WHEN_EXHAUSTED_BLOCK} (otherwise ignored) (see {@link #setMaxWait}) + * @param maxIdle the maximum number of idle objects in my pool (see {@link #setMaxIdle}) + * @param minIdle the minimum number of idle objects in my pool (see {@link #setMinIdle}) + * @param testOnBorrow whether or not to validate objects before they are returned by the {@link #borrowObject} method (see {@link #setTestOnBorrow}) + * @param testOnReturn whether or not to validate objects after they are returned to the {@link #returnObject} method (see {@link #setTestOnReturn}) + * @param timeBetweenEvictionRunsMillis the amount of time (in milliseconds) to sleep between examining idle objects for eviction (see {@link #setTimeBetweenEvictionRunsMillis}) + * @param numTestsPerEvictionRun the number of idle objects to examine per run within the idle object eviction thread (if any) (see {@link #setNumTestsPerEvictionRun}) + * @param minEvictableIdleTimeMillis the minimum number of milliseconds an object can sit idle in the pool before it is eligable for evcition (see {@link #setMinEvictableIdleTimeMillis}) + * @param testWhileIdle whether or not to validate objects in the idle object eviction thread, if any (see {@link #setTestWhileIdle}) + * @param softMinEvictableIdleTimeMillis the minimum number of milliseconds an object can sit idle in the pool before it is eligable for evcition with the extra condition that at least "minIdle" amount of object remain in the pool. (see {@link #setSoftMinEvictableIdleTimeMillis}) + */ + public FairGenericObjectPool(PoolableObjectFactory factory, int maxActive, byte whenExhaustedAction, long maxWait, int maxIdle, int minIdle, boolean testOnBorrow, boolean testOnReturn, long timeBetweenEvictionRunsMillis, int numTestsPerEvictionRun, long minEvictableIdleTimeMillis, boolean testWhileIdle, long softMinEvictableIdleTimeMillis) { + super(factory, maxActive, whenExhaustedAction, maxWait, maxIdle, + minIdle, testOnBorrow, testOnReturn, + timeBetweenEvictionRunsMillis, numTestsPerEvictionRun, + minEvictableIdleTimeMillis, testWhileIdle, + softMinEvictableIdleTimeMillis); + _borrowerQueue = Collections.synchronizedList(new LinkedList()); + } + + //-- ObjectPool methods ------------------------------------------ + + /** + * + * @see org.apache.commons.pool.ObjectPool#borrowObject() + */ + public Object borrowObject() throws Exception { + assertOpen(); + long starttime = System.currentTimeMillis(); + + + + try { + synchronized(this) { + // use borrowerQueue + _borrowerQueue.add(Thread.currentThread()); + + for(;;) { + ObjectTimestampPair pair = null; + + // Only allow current thread to receive pool object if + // thread is top of queue + boolean eligible = _borrowerQueue.get(0)==Thread.currentThread(); + if(eligible) { + // if there are any sleeping, just grab one of those + try { + pair = (ObjectTimestampPair)(_pool.removeFirst()); + } catch(NoSuchElementException e) { + ; /* ignored */ + } + } + + // otherwise + if(null == pair) { + // check if we can create one + // (note we know that the num sleeping is 0, else we wouldn't be here) + if(eligible && (_maxActive < 0 || _numActive < _maxActive)) { + // allow new object to be created + } else { + // the pool is exhausted + // or current thread is ineligible due to fairness + switch(_whenExhaustedAction) { + case WHEN_EXHAUSTED_GROW: + // allow new object to be created + break; + case WHEN_EXHAUSTED_FAIL: + throw new NoSuchElementException("Pool exhausted"); + case WHEN_EXHAUSTED_BLOCK: + try { + if(_maxWait <= 0) { + wait(); + } else { + // this code may be executed again after a notify then continue cycle + // so, need to calculate the amount of time to wait + final long elapsed = (System.currentTimeMillis() - starttime); + final long waitTime = _maxWait - elapsed; + if (waitTime > 0) + { + wait(waitTime); + } + } + } catch(InterruptedException e) { + // ignored + } + if(_maxWait > 0 && ((System.currentTimeMillis() - starttime) >= _maxWait)) { + throw new NoSuchElementException("Timeout waiting for idle object"); + } else { + continue; // keep looping + } + default: + throw new IllegalArgumentException("WhenExhaustedAction property " + _whenExhaustedAction + " not recognized."); + } + } + } + _numActive++; + + // create new object when needed + boolean newlyCreated = false; + if(null == pair) { + try { + Object obj = _factory.makeObject(); + pair = new ObjectTimestampPair(obj); + newlyCreated = true; + return pair.value; + } finally { + if (!newlyCreated) { + // object cannot be created + _numActive--; + notifyAll(); + } + } + } + + // activate & validate the object + try { + _factory.activateObject(pair.value); + if(_testOnBorrow && !_factory.validateObject(pair.value)) { + throw new Exception("ValidateObject failed"); + } + return pair.value; + } + catch (Throwable e) { + // object cannot be activated or is invalid + _numActive--; + notifyAll(); + try { + _factory.destroyObject(pair.value); + } + catch (Throwable e2) { + // cannot destroy broken object + } + if(newlyCreated) { + throw new NoSuchElementException("Could not create a validated object, cause: " + e.getMessage()); + } + else { + continue; // keep looping + } + } + } + } + } finally { + // remove thread from queue on any method exit + _borrowerQueue.remove(Thread.currentThread()); + } + } + + /** Waiting borrowers (threads in #borrowObject ) */ + protected List _borrowerQueue; +} diff --git a/commons/src/main/java/org/apache/commons/pool/impl/FairGenericObjectPoolTest.java b/commons/src/main/java/org/apache/commons/pool/impl/FairGenericObjectPoolTest.java new file mode 100644 index 00000000..c9ce73ac --- /dev/null +++ b/commons/src/main/java/org/apache/commons/pool/impl/FairGenericObjectPoolTest.java @@ -0,0 +1,122 @@ +/* FairGenericObjectPoolTest +* +* $Id$ +* +* Created on Apr 7, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +*/ +package org.apache.commons.pool.impl; + +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; + +import junit.framework.TestCase; + +import org.apache.commons.pool.BasePoolableObjectFactory; + +/** + * Test for FairGenericObjectPool. + * + * @author gojomo + */ +@SuppressWarnings("unchecked") +public class FairGenericObjectPoolTest extends TestCase { +// public void testUnfair() throws InterruptedException { +//// System.out.println("unfair"); +// GenericObjectPool pool = new GenericObjectPool(); +// +// Object[] borrowOrder = tryPool(pool); +// +// Object[] sortedOrder = (Object[]) borrowOrder.clone(); +// Arrays.sort(sortedOrder); +// assertFalse("unexpectedly fair", Arrays.equals(borrowOrder,sortedOrder)); +// } + + public void testFair() throws InterruptedException { +// System.out.println("fair"); + GenericObjectPool pool = new FairGenericObjectPool(); + + Object[] borrowOrder = tryPool(pool); + + Object[] sortedOrder = (Object[]) borrowOrder.clone(); + Arrays.sort(sortedOrder); + assertTrue("unexpectedly unfair", Arrays.equals(borrowOrder,sortedOrder)); + } + + /** + * Test the given pool for fairness. + * + * @param pool GenericObjectPool to test + * @throws InterruptedException + */ + private Object[] tryPool(GenericObjectPool pool) throws InterruptedException { + BlockerObjectFactory factory = new BlockerObjectFactory(); + pool.setFactory(factory); + pool.setMaxActive(1); + List borrowOrder = Collections.synchronizedList(new LinkedList()); + for(int i = 0; i < 10; i++) { + Contender c = new Contender(borrowOrder); + c.pool = pool; + c.ordinal = i; + (new Thread(c)).start(); + Thread.sleep(500); + } + factory.single.release(); + Thread.sleep(5000); + return borrowOrder.toArray(); + } + + class Contender implements Runnable { + public GenericObjectPool pool; + public int ordinal; + public List reportList; + + public Contender(List borrowOrder) { + reportList = borrowOrder; + } + + public void run() { + try { + Blocker block = (Blocker) pool.borrowObject(); + System.out.println("borrowed #"+ordinal); + reportList.add(new Integer(ordinal)); + block.acquire(); + System.out.println("returning #"+ordinal); + pool.returnObject(block); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + } + + class BlockerObjectFactory extends BasePoolableObjectFactory { + public Blocker single = new Blocker(); + public Object makeObject() throws Exception { + System.out.println("makeObject"); + return single; + } + } + + class Blocker { + boolean block = true; + public synchronized void acquire() { + // only block first time through + if(block) { + try { + wait(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + block = false; + } + public synchronized void release() { + notifyAll(); + } + } +} diff --git a/commons/src/main/java/org/apache/commons/pool/impl/GenericObjectPool.java b/commons/src/main/java/org/apache/commons/pool/impl/GenericObjectPool.java new file mode 100644 index 00000000..45dcbbec --- /dev/null +++ b/commons/src/main/java/org/apache/commons/pool/impl/GenericObjectPool.java @@ -0,0 +1,1312 @@ +/* + * Copyright 1999-2004 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.commons.pool.impl; + +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.LinkedList; +import java.util.ListIterator; +import java.util.Timer; +import java.util.TimerTask; + +import org.apache.commons.pool.BaseObjectPool; +import org.apache.commons.pool.ObjectPool; +import org.apache.commons.pool.PoolableObjectFactory; +import org.apache.commons.pool.impl.GenericKeyedObjectPool.ObjectTimestampPair; + +/** + * A configurable {@link ObjectPool} implementation. + *

+ * When coupled with the appropriate {@link PoolableObjectFactory}, + * GenericObjectPool provides robust pooling functionality for + * arbitrary objects. + *

+ * A GenericObjectPool provides a number of configurable parameters: + *

    + *
  • + * {@link #setMaxActive maxActive} controls the maximum number of objects that can + * be borrowed from the pool at one time. When non-positive, there + * is no limit to the number of objects that may be active at one time. + * When {@link #setMaxActive maxActive} is exceeded, the pool is said to be exhausted. + *
    • + *
  • + * {@link #setMaxIdle maxIdle} controls the maximum number of objects that can + * sit idle in the pool at any time. When negative, there + * is no limit to the number of objects that may be idle at one time. + *
    • + *
  • + * {@link #setWhenExhaustedAction whenExhaustedAction} specifies the + * behaviour of the {@link #borrowObject} method when the pool is exhausted: + *
      + *
    • + * When {@link #setWhenExhaustedAction whenExhaustedAction} is + * {@link #WHEN_EXHAUSTED_FAIL}, {@link #borrowObject} will throw + * a {@link NoSuchElementException} + *
      • + *
    • + * When {@link #setWhenExhaustedAction whenExhaustedAction} is + * {@link #WHEN_EXHAUSTED_GROW}, {@link #borrowObject} will create a new + * object and return it(essentially making {@link #setMaxActive maxActive} + * meaningless.) + *
      • + *
    • + * When {@link #setWhenExhaustedAction whenExhaustedAction} + * is {@link #WHEN_EXHAUSTED_BLOCK}, {@link #borrowObject} will block + * (invoke {@link Object#wait} until a new or idle object is available. + * If a positive {@link #setMaxWait maxWait} + * value is supplied, the {@link #borrowObject} will block for at + * most that many milliseconds, after which a {@link NoSuchElementException} + * will be thrown. If {@link #setMaxWait maxWait} is non-positive, + * the {@link #borrowObject} method will block indefinitely. + *
      • + *
    + *
    • + *
  • + * When {@link #setTestOnBorrow testOnBorrow} is set, the pool will + * attempt to validate each object before it is returned from the + * {@link #borrowObject} method. (Using the provided factory's + * {@link PoolableObjectFactory#validateObject} method.) Objects that fail + * to validate will be dropped from the pool, and a different object will + * be borrowed. + *
    • + *
  • + * When {@link #setTestOnReturn testOnReturn} is set, the pool will + * attempt to validate each object before it is returned to the pool in the + * {@link #returnObject} method. (Using the provided factory's + * {@link PoolableObjectFactory#validateObject} + * method.) Objects that fail to validate will be dropped from the pool. + *
    • + *
+ *

+ * Optionally, one may configure the pool to examine and possibly evict objects as they + * sit idle in the pool. This is performed by an "idle object eviction" thread, which + * runs asychronously. The idle object eviction thread may be configured using the + * following attributes: + *

    + *
  • + * {@link #setTimeBetweenEvictionRunsMillis timeBetweenEvictionRunsMillis} + * indicates how long the eviction thread should sleep before "runs" of examining + * idle objects. When non-positive, no eviction thread will be launched. + *
    • + *
  • + * {@link #setMinEvictableIdleTimeMillis minEvictableIdleTimeMillis} + * specifies the minimum amount of time that an object may sit idle in the pool + * before it is eligable for eviction due to idle time. When non-positive, no object + * will be dropped from the pool due to idle time alone. + *
    • + *
  • + * {@link #setTestWhileIdle testWhileIdle} indicates whether or not idle + * objects should be validated using the factory's + * {@link PoolableObjectFactory#validateObject} method. Objects + * that fail to validate will be dropped from the pool. + *
    • + *
+ *

+ * GenericObjectPool is not usable without a {@link PoolableObjectFactory}. A + * non-null factory must be provided either as a constructor argument + * or via a call to {@link #setFactory} before the pool is used. + * + * @see GenericKeyedObjectPool + * @author Rodney Waldhoff + * @author Dirk Verbeeck + * @version $Revision$ $Date$ + */ +@SuppressWarnings("unchecked") +public class GenericObjectPool extends BaseObjectPool implements ObjectPool { + + //--- public constants ------------------------------------------- + + /** + * A "when exhausted action" type indicating that when the pool is + * exhausted (i.e., the maximum number of active objects has + * been reached), the {@link #borrowObject} + * method should fail, throwing a {@link NoSuchElementException}. + * @see #WHEN_EXHAUSTED_BLOCK + * @see #WHEN_EXHAUSTED_GROW + * @see #setWhenExhaustedAction + */ + public static final byte WHEN_EXHAUSTED_FAIL = 0; + + /** + * A "when exhausted action" type indicating that when the pool + * is exhausted (i.e., the maximum number + * of active objects has been reached), the {@link #borrowObject} + * method should block until a new object is available, or the + * {@link #getMaxWait maximum wait time} has been reached. + * @see #WHEN_EXHAUSTED_FAIL + * @see #WHEN_EXHAUSTED_GROW + * @see #setMaxWait + * @see #getMaxWait + * @see #setWhenExhaustedAction + */ + public static final byte WHEN_EXHAUSTED_BLOCK = 1; + + /** + * A "when exhausted action" type indicating that when the pool is + * exhausted (i.e., the maximum number + * of active objects has been reached), the {@link #borrowObject} + * method should simply create a new object anyway. + * @see #WHEN_EXHAUSTED_FAIL + * @see #WHEN_EXHAUSTED_GROW + * @see #setWhenExhaustedAction + */ + public static final byte WHEN_EXHAUSTED_GROW = 2; + + /** + * The default cap on the number of "sleeping" instances in the pool. + * @see #getMaxIdle + * @see #setMaxIdle + */ + public static final int DEFAULT_MAX_IDLE = 8; + + /** + * The default minimum number of "sleeping" instances in the pool + * before before the evictor thread (if active) spawns new objects. + * @see #getMinIdle + * @see #setMinIdle + */ + public static final int DEFAULT_MIN_IDLE = 0; + + /** + * The default cap on the total number of active instances from the pool. + * @see #getMaxActive + */ + public static final int DEFAULT_MAX_ACTIVE = 8; + + /** + * The default "when exhausted action" for the pool. + * @see #WHEN_EXHAUSTED_BLOCK + * @see #WHEN_EXHAUSTED_FAIL + * @see #WHEN_EXHAUSTED_GROW + * @see #setWhenExhaustedAction + */ + public static final byte DEFAULT_WHEN_EXHAUSTED_ACTION = WHEN_EXHAUSTED_BLOCK; + + /** + * The default maximum amount of time (in millis) the + * {@link #borrowObject} method should block before throwing + * an exception when the pool is exhausted and the + * {@link #getWhenExhaustedAction "when exhausted" action} is + * {@link #WHEN_EXHAUSTED_BLOCK}. + * @see #getMaxWait + * @see #setMaxWait + */ + public static final long DEFAULT_MAX_WAIT = -1L; + + /** + * The default "test on borrow" value. + * @see #getTestOnBorrow + * @see #setTestOnBorrow + */ + public static final boolean DEFAULT_TEST_ON_BORROW = false; + + /** + * The default "test on return" value. + * @see #getTestOnReturn + * @see #setTestOnReturn + */ + public static final boolean DEFAULT_TEST_ON_RETURN = false; + + /** + * The default "test while idle" value. + * @see #getTestWhileIdle + * @see #setTestWhileIdle + * @see #getTimeBetweenEvictionRunsMillis + * @see #setTimeBetweenEvictionRunsMillis + */ + public static final boolean DEFAULT_TEST_WHILE_IDLE = false; + + /** + * The default "time between eviction runs" value. + * @see #getTimeBetweenEvictionRunsMillis + * @see #setTimeBetweenEvictionRunsMillis + */ + public static final long DEFAULT_TIME_BETWEEN_EVICTION_RUNS_MILLIS = -1L; + + /** + * The default number of objects to examine per run in the + * idle object evictor. + * @see #getNumTestsPerEvictionRun + * @see #setNumTestsPerEvictionRun + * @see #getTimeBetweenEvictionRunsMillis + * @see #setTimeBetweenEvictionRunsMillis + */ + public static final int DEFAULT_NUM_TESTS_PER_EVICTION_RUN = 3; + + /** + * The default value for {@link #getMinEvictableIdleTimeMillis}. + * @see #getMinEvictableIdleTimeMillis + * @see #setMinEvictableIdleTimeMillis + */ + public static final long DEFAULT_MIN_EVICTABLE_IDLE_TIME_MILLIS = 1000L * 60L * 30L; + + /** + * The default value for {@link #getSoftMinEvictableIdleTimeMillis}. + * @see #getSoftMinEvictableIdleTimeMillis + * @see #setSoftMinEvictableIdleTimeMillis + */ + public static final long DEFAULT_SOFT_MIN_EVICTABLE_IDLE_TIME_MILLIS = -1; + + //--- package constants ------------------------------------------- + + /** + * Idle object evition Timer. Shared between all {@link GenericObjectPool}s and {@link GenericKeyedObjectPool} s. + */ + static final Timer EVICTION_TIMER = new Timer(true); + + //--- constructors ----------------------------------------------- + + /** + * Create a new GenericObjectPool. + */ + public GenericObjectPool() { + this(null,DEFAULT_MAX_ACTIVE,DEFAULT_WHEN_EXHAUSTED_ACTION,DEFAULT_MAX_WAIT,DEFAULT_MAX_IDLE,DEFAULT_MIN_IDLE,DEFAULT_TEST_ON_BORROW,DEFAULT_TEST_ON_RETURN,DEFAULT_TIME_BETWEEN_EVICTION_RUNS_MILLIS,DEFAULT_NUM_TESTS_PER_EVICTION_RUN,DEFAULT_MIN_EVICTABLE_IDLE_TIME_MILLIS,DEFAULT_TEST_WHILE_IDLE); + } + + /** + * Create a new GenericObjectPool using the specified values. + * @param factory the (possibly null)PoolableObjectFactory to use to create, validate and destroy objects + */ + public GenericObjectPool(PoolableObjectFactory factory) { + this(factory,DEFAULT_MAX_ACTIVE,DEFAULT_WHEN_EXHAUSTED_ACTION,DEFAULT_MAX_WAIT,DEFAULT_MAX_IDLE,DEFAULT_MIN_IDLE,DEFAULT_TEST_ON_BORROW,DEFAULT_TEST_ON_RETURN,DEFAULT_TIME_BETWEEN_EVICTION_RUNS_MILLIS,DEFAULT_NUM_TESTS_PER_EVICTION_RUN,DEFAULT_MIN_EVICTABLE_IDLE_TIME_MILLIS,DEFAULT_TEST_WHILE_IDLE); + } + + /** + * Create a new GenericObjectPool using the specified values. + * @param factory the (possibly null)PoolableObjectFactory to use to create, validate and destroy objects + * @param config a non-null {@link GenericObjectPool.Config} describing my configuration + */ + public GenericObjectPool(PoolableObjectFactory factory, GenericObjectPool.Config config) { + this(factory,config.maxActive,config.whenExhaustedAction,config.maxWait,config.maxIdle,config.minIdle,config.testOnBorrow,config.testOnReturn,config.timeBetweenEvictionRunsMillis,config.numTestsPerEvictionRun,config.minEvictableIdleTimeMillis,config.testWhileIdle); + } + + /** + * Create a new GenericObjectPool using the specified values. + * @param factory the (possibly null)PoolableObjectFactory to use to create, validate and destroy objects + * @param maxActive the maximum number of objects that can be borrowed from me at one time (see {@link #setMaxActive}) + */ + public GenericObjectPool(PoolableObjectFactory factory, int maxActive) { + this(factory,maxActive,DEFAULT_WHEN_EXHAUSTED_ACTION,DEFAULT_MAX_WAIT,DEFAULT_MAX_IDLE,DEFAULT_MIN_IDLE,DEFAULT_TEST_ON_BORROW,DEFAULT_TEST_ON_RETURN,DEFAULT_TIME_BETWEEN_EVICTION_RUNS_MILLIS,DEFAULT_NUM_TESTS_PER_EVICTION_RUN,DEFAULT_MIN_EVICTABLE_IDLE_TIME_MILLIS,DEFAULT_TEST_WHILE_IDLE); + } + + /** + * Create a new GenericObjectPool using the specified values. + * @param factory the (possibly null)PoolableObjectFactory to use to create, validate and destroy objects + * @param maxActive the maximum number of objects that can be borrowed from me at one time (see {@link #setMaxActive}) + * @param whenExhaustedAction the action to take when the pool is exhausted (see {@link #getWhenExhaustedAction}) + * @param maxWait the maximum amount of time to wait for an idle object when the pool is exhausted an and whenExhaustedAction is {@link #WHEN_EXHAUSTED_BLOCK} (otherwise ignored) (see {@link #getMaxWait}) + */ + public GenericObjectPool(PoolableObjectFactory factory, int maxActive, byte whenExhaustedAction, long maxWait) { + this(factory,maxActive,whenExhaustedAction,maxWait,DEFAULT_MAX_IDLE,DEFAULT_MIN_IDLE,DEFAULT_TEST_ON_BORROW,DEFAULT_TEST_ON_RETURN,DEFAULT_TIME_BETWEEN_EVICTION_RUNS_MILLIS,DEFAULT_NUM_TESTS_PER_EVICTION_RUN,DEFAULT_MIN_EVICTABLE_IDLE_TIME_MILLIS,DEFAULT_TEST_WHILE_IDLE); + } + + /** + * Create a new GenericObjectPool using the specified values. + * @param factory the (possibly null)PoolableObjectFactory to use to create, validate and destroy objects + * @param maxActive the maximum number of objects that can be borrowed from me at one time (see {@link #setMaxActive}) + * @param whenExhaustedAction the action to take when the pool is exhausted (see {@link #getWhenExhaustedAction}) + * @param maxWait the maximum amount of time to wait for an idle object when the pool is exhausted an and whenExhaustedAction is {@link #WHEN_EXHAUSTED_BLOCK} (otherwise ignored) (see {@link #getMaxWait}) + * @param testOnBorrow whether or not to validate objects before they are returned by the {@link #borrowObject} method (see {@link #getTestOnBorrow}) + * @param testOnReturn whether or not to validate objects after they are returned to the {@link #returnObject} method (see {@link #getTestOnReturn}) + */ + public GenericObjectPool(PoolableObjectFactory factory, int maxActive, byte whenExhaustedAction, long maxWait, boolean testOnBorrow, boolean testOnReturn) { + this(factory,maxActive,whenExhaustedAction,maxWait,DEFAULT_MAX_IDLE,DEFAULT_MIN_IDLE,testOnBorrow,testOnReturn,DEFAULT_TIME_BETWEEN_EVICTION_RUNS_MILLIS,DEFAULT_NUM_TESTS_PER_EVICTION_RUN,DEFAULT_MIN_EVICTABLE_IDLE_TIME_MILLIS,DEFAULT_TEST_WHILE_IDLE); + } + + /** + * Create a new GenericObjectPool using the specified values. + * @param factory the (possibly null)PoolableObjectFactory to use to create, validate and destroy objects + * @param maxActive the maximum number of objects that can be borrowed from me at one time (see {@link #setMaxActive}) + * @param whenExhaustedAction the action to take when the pool is exhausted (see {@link #getWhenExhaustedAction}) + * @param maxWait the maximum amount of time to wait for an idle object when the pool is exhausted an and whenExhaustedAction is {@link #WHEN_EXHAUSTED_BLOCK} (otherwise ignored) (see {@link #getMaxWait}) + * @param maxIdle the maximum number of idle objects in my pool (see {@link #getMaxIdle}) + */ + public GenericObjectPool(PoolableObjectFactory factory, int maxActive, byte whenExhaustedAction, long maxWait, int maxIdle) { + this(factory,maxActive,whenExhaustedAction,maxWait,maxIdle,DEFAULT_MIN_IDLE,DEFAULT_TEST_ON_BORROW,DEFAULT_TEST_ON_RETURN,DEFAULT_TIME_BETWEEN_EVICTION_RUNS_MILLIS,DEFAULT_NUM_TESTS_PER_EVICTION_RUN,DEFAULT_MIN_EVICTABLE_IDLE_TIME_MILLIS,DEFAULT_TEST_WHILE_IDLE); + } + + /** + * Create a new GenericObjectPool using the specified values. + * @param factory the (possibly null)PoolableObjectFactory to use to create, validate and destroy objects + * @param maxActive the maximum number of objects that can be borrowed from me at one time (see {@link #setMaxActive}) + * @param whenExhaustedAction the action to take when the pool is exhausted (see {@link #getWhenExhaustedAction}) + * @param maxWait the maximum amount of time to wait for an idle object when the pool is exhausted an and whenExhaustedAction is {@link #WHEN_EXHAUSTED_BLOCK} (otherwise ignored) (see {@link #getMaxWait}) + * @param maxIdle the maximum number of idle objects in my pool (see {@link #getMaxIdle}) + * @param testOnBorrow whether or not to validate objects before they are returned by the {@link #borrowObject} method (see {@link #getTestOnBorrow}) + * @param testOnReturn whether or not to validate objects after they are returned to the {@link #returnObject} method (see {@link #getTestOnReturn}) + */ + public GenericObjectPool(PoolableObjectFactory factory, int maxActive, byte whenExhaustedAction, long maxWait, int maxIdle, boolean testOnBorrow, boolean testOnReturn) { + this(factory,maxActive,whenExhaustedAction,maxWait,maxIdle,DEFAULT_MIN_IDLE,testOnBorrow,testOnReturn,DEFAULT_TIME_BETWEEN_EVICTION_RUNS_MILLIS,DEFAULT_NUM_TESTS_PER_EVICTION_RUN,DEFAULT_MIN_EVICTABLE_IDLE_TIME_MILLIS,DEFAULT_TEST_WHILE_IDLE); + } + + /** + * Create a new GenericObjectPool using the specified values. + * @param factory the (possibly null)PoolableObjectFactory to use to create, validate and destroy objects + * @param maxActive the maximum number of objects that can be borrowed from me at one time (see {@link #setMaxActive}) + * @param whenExhaustedAction the action to take when the pool is exhausted (see {@link #setWhenExhaustedAction}) + * @param maxWait the maximum amount of time to wait for an idle object when the pool is exhausted an and whenExhaustedAction is {@link #WHEN_EXHAUSTED_BLOCK} (otherwise ignored) (see {@link #setMaxWait}) + * @param maxIdle the maximum number of idle objects in my pool (see {@link #setMaxIdle}) + * @param testOnBorrow whether or not to validate objects before they are returned by the {@link #borrowObject} method (see {@link #setTestOnBorrow}) + * @param testOnReturn whether or not to validate objects after they are returned to the {@link #returnObject} method (see {@link #setTestOnReturn}) + * @param timeBetweenEvictionRunsMillis the amount of time (in milliseconds) to sleep between examining idle objects for eviction (see {@link #setTimeBetweenEvictionRunsMillis}) + * @param numTestsPerEvictionRun the number of idle objects to examine per run within the idle object eviction thread (if any) (see {@link #setNumTestsPerEvictionRun}) + * @param minEvictableIdleTimeMillis the minimum number of milliseconds an object can sit idle in the pool before it is eligable for evcition (see {@link #setMinEvictableIdleTimeMillis}) + * @param testWhileIdle whether or not to validate objects in the idle object eviction thread, if any (see {@link #setTestWhileIdle}) + */ + public GenericObjectPool(PoolableObjectFactory factory, int maxActive, byte whenExhaustedAction, long maxWait, int maxIdle, boolean testOnBorrow, boolean testOnReturn, long timeBetweenEvictionRunsMillis, int numTestsPerEvictionRun, long minEvictableIdleTimeMillis, boolean testWhileIdle) { + this(factory, maxActive, whenExhaustedAction, maxWait, maxIdle, DEFAULT_MIN_IDLE, testOnBorrow, testOnReturn, timeBetweenEvictionRunsMillis, numTestsPerEvictionRun, minEvictableIdleTimeMillis, testWhileIdle); + } + + /** + * Create a new GenericObjectPool using the specified values. + * @param factory the (possibly null)PoolableObjectFactory to use to create, validate and destroy objects + * @param maxActive the maximum number of objects that can be borrowed from me at one time (see {@link #setMaxActive}) + * @param whenExhaustedAction the action to take when the pool is exhausted (see {@link #setWhenExhaustedAction}) + * @param maxWait the maximum amount of time to wait for an idle object when the pool is exhausted an and whenExhaustedAction is {@link #WHEN_EXHAUSTED_BLOCK} (otherwise ignored) (see {@link #setMaxWait}) + * @param maxIdle the maximum number of idle objects in my pool (see {@link #setMaxIdle}) + * @param minIdle the minimum number of idle objects in my pool (see {@link #setMinIdle}) + * @param testOnBorrow whether or not to validate objects before they are returned by the {@link #borrowObject} method (see {@link #setTestOnBorrow}) + * @param testOnReturn whether or not to validate objects after they are returned to the {@link #returnObject} method (see {@link #setTestOnReturn}) + * @param timeBetweenEvictionRunsMillis the amount of time (in milliseconds) to sleep between examining idle objects for eviction (see {@link #setTimeBetweenEvictionRunsMillis}) + * @param numTestsPerEvictionRun the number of idle objects to examine per run within the idle object eviction thread (if any) (see {@link #setNumTestsPerEvictionRun}) + * @param minEvictableIdleTimeMillis the minimum number of milliseconds an object can sit idle in the pool before it is eligable for evcition (see {@link #setMinEvictableIdleTimeMillis}) + * @param testWhileIdle whether or not to validate objects in the idle object eviction thread, if any (see {@link #setTestWhileIdle}) + */ + public GenericObjectPool(PoolableObjectFactory factory, int maxActive, byte whenExhaustedAction, long maxWait, int maxIdle, int minIdle, boolean testOnBorrow, boolean testOnReturn, long timeBetweenEvictionRunsMillis, int numTestsPerEvictionRun, long minEvictableIdleTimeMillis, boolean testWhileIdle) { + this(factory, maxActive, whenExhaustedAction, maxWait, maxIdle, minIdle, testOnBorrow, testOnReturn, timeBetweenEvictionRunsMillis, numTestsPerEvictionRun, minEvictableIdleTimeMillis, testWhileIdle, DEFAULT_SOFT_MIN_EVICTABLE_IDLE_TIME_MILLIS); + } + + /** + * Create a new GenericObjectPool using the specified values. + * @param factory the (possibly null)PoolableObjectFactory to use to create, validate and destroy objects + * @param maxActive the maximum number of objects that can be borrowed from me at one time (see {@link #setMaxActive}) + * @param whenExhaustedAction the action to take when the pool is exhausted (see {@link #setWhenExhaustedAction}) + * @param maxWait the maximum amount of time to wait for an idle object when the pool is exhausted an and whenExhaustedAction is {@link #WHEN_EXHAUSTED_BLOCK} (otherwise ignored) (see {@link #setMaxWait}) + * @param maxIdle the maximum number of idle objects in my pool (see {@link #setMaxIdle}) + * @param minIdle the minimum number of idle objects in my pool (see {@link #setMinIdle}) + * @param testOnBorrow whether or not to validate objects before they are returned by the {@link #borrowObject} method (see {@link #setTestOnBorrow}) + * @param testOnReturn whether or not to validate objects after they are returned to the {@link #returnObject} method (see {@link #setTestOnReturn}) + * @param timeBetweenEvictionRunsMillis the amount of time (in milliseconds) to sleep between examining idle objects for eviction (see {@link #setTimeBetweenEvictionRunsMillis}) + * @param numTestsPerEvictionRun the number of idle objects to examine per run within the idle object eviction thread (if any) (see {@link #setNumTestsPerEvictionRun}) + * @param minEvictableIdleTimeMillis the minimum number of milliseconds an object can sit idle in the pool before it is eligable for evcition (see {@link #setMinEvictableIdleTimeMillis}) + * @param testWhileIdle whether or not to validate objects in the idle object eviction thread, if any (see {@link #setTestWhileIdle}) + * @param softMinEvictableIdleTimeMillis the minimum number of milliseconds an object can sit idle in the pool before it is eligable for evcition with the extra condition that at least "minIdle" amount of object remain in the pool. (see {@link #setSoftMinEvictableIdleTimeMillis}) + */ + public GenericObjectPool(PoolableObjectFactory factory, int maxActive, byte whenExhaustedAction, long maxWait, int maxIdle, int minIdle, boolean testOnBorrow, boolean testOnReturn, long timeBetweenEvictionRunsMillis, int numTestsPerEvictionRun, long minEvictableIdleTimeMillis, boolean testWhileIdle, long softMinEvictableIdleTimeMillis) { + _factory = factory; + _maxActive = maxActive; + switch(whenExhaustedAction) { + case WHEN_EXHAUSTED_BLOCK: + case WHEN_EXHAUSTED_FAIL: + case WHEN_EXHAUSTED_GROW: + _whenExhaustedAction = whenExhaustedAction; + break; + default: + throw new IllegalArgumentException("whenExhaustedAction " + whenExhaustedAction + " not recognized."); + } + _maxWait = maxWait; + _maxIdle = maxIdle; + _minIdle = minIdle; + _testOnBorrow = testOnBorrow; + _testOnReturn = testOnReturn; + _timeBetweenEvictionRunsMillis = timeBetweenEvictionRunsMillis; + _numTestsPerEvictionRun = numTestsPerEvictionRun; + _minEvictableIdleTimeMillis = minEvictableIdleTimeMillis; + _softMinEvictableIdleTimeMillis = softMinEvictableIdleTimeMillis; + _testWhileIdle = testWhileIdle; + + _pool = new LinkedList(); + startEvictor(_timeBetweenEvictionRunsMillis); + } + + //--- public methods --------------------------------------------- + + //--- configuration methods -------------------------------------- + + /** + * Returns the cap on the total number of active instances from my pool. + * @return the cap on the total number of active instances from my pool. + * @see #setMaxActive + */ + public synchronized int getMaxActive() { + return _maxActive; + } + + /** + * Sets the cap on the total number of active instances from my pool. + * @param maxActive The cap on the total number of active instances from my pool. + * Use a negative value for an infinite number of instances. + * @see #getMaxActive + */ + public synchronized void setMaxActive(int maxActive) { + _maxActive = maxActive; + notifyAll(); + } + + /** + * Returns the action to take when the {@link #borrowObject} method + * is invoked when the pool is exhausted (the maximum number + * of "active" objects has been reached). + * + * @return one of {@link #WHEN_EXHAUSTED_BLOCK}, {@link #WHEN_EXHAUSTED_FAIL} or {@link #WHEN_EXHAUSTED_GROW} + * @see #setWhenExhaustedAction + */ + public synchronized byte getWhenExhaustedAction() { + return _whenExhaustedAction; + } + + /** + * Sets the action to take when the {@link #borrowObject} method + * is invoked when the pool is exhausted (the maximum number + * of "active" objects has been reached). + * + * @param whenExhaustedAction the action code, which must be one of + * {@link #WHEN_EXHAUSTED_BLOCK}, {@link #WHEN_EXHAUSTED_FAIL}, + * or {@link #WHEN_EXHAUSTED_GROW} + * @see #getWhenExhaustedAction + */ + public synchronized void setWhenExhaustedAction(byte whenExhaustedAction) { + switch(whenExhaustedAction) { + case WHEN_EXHAUSTED_BLOCK: + case WHEN_EXHAUSTED_FAIL: + case WHEN_EXHAUSTED_GROW: + _whenExhaustedAction = whenExhaustedAction; + notifyAll(); + break; + default: + throw new IllegalArgumentException("whenExhaustedAction " + whenExhaustedAction + " not recognized."); + } + } + + + /** + * Returns the maximum amount of time (in milliseconds) the + * {@link #borrowObject} method should block before throwing + * an exception when the pool is exhausted and the + * {@link #setWhenExhaustedAction "when exhausted" action} is + * {@link #WHEN_EXHAUSTED_BLOCK}. + * + * When less than 0, the {@link #borrowObject} method + * may block indefinitely. + * + * @see #setMaxWait + * @see #setWhenExhaustedAction + * @see #WHEN_EXHAUSTED_BLOCK + */ + public synchronized long getMaxWait() { + return _maxWait; + } + + /** + * Sets the maximum amount of time (in milliseconds) the + * {@link #borrowObject} method should block before throwing + * an exception when the pool is exhausted and the + * {@link #setWhenExhaustedAction "when exhausted" action} is + * {@link #WHEN_EXHAUSTED_BLOCK}. + * + * When less than 0, the {@link #borrowObject} method + * may block indefinitely. + * + * @see #getMaxWait + * @see #setWhenExhaustedAction + * @see #WHEN_EXHAUSTED_BLOCK + */ + public synchronized void setMaxWait(long maxWait) { + _maxWait = maxWait; + notifyAll(); + } + + /** + * Returns the cap on the number of "idle" instances in the pool. + * @return the cap on the number of "idle" instances in the pool. + * @see #setMaxIdle + */ + public synchronized int getMaxIdle() { + return _maxIdle; + } + + /** + * Sets the cap on the number of "idle" instances in the pool. + * @param maxIdle The cap on the number of "idle" instances in the pool. + * Use a negative value to indicate an unlimited number + * of idle instances. + * @see #getMaxIdle + */ + public synchronized void setMaxIdle(int maxIdle) { + _maxIdle = maxIdle; + notifyAll(); + } + + /** + * Sets the minimum number of objects allowed in the pool + * before the evictor thread (if active) spawns new objects. + * (Note no objects are created when: numActive + numIdle >= maxActive) + * + * @param minIdle The minimum number of objects. + * @see #getMinIdle + */ + public synchronized void setMinIdle(int minIdle) { + _minIdle = minIdle; + notifyAll(); + } + + /** + * Returns the minimum number of objects allowed in the pool + * before the evictor thread (if active) spawns new objects. + * (Note no objects are created when: numActive + numIdle >= maxActive) + * + * @return The minimum number of objects. + * @see #setMinIdle + */ + public synchronized int getMinIdle() { + return _minIdle; + } + + /** + * When true, objects will be + * {@link PoolableObjectFactory#validateObject validated} + * before being returned by the {@link #borrowObject} + * method. If the object fails to validate, + * it will be dropped from the pool, and we will attempt + * to borrow another. + * + * @see #setTestOnBorrow + */ + public synchronized boolean getTestOnBorrow() { + return _testOnBorrow; + } + + /** + * When true, objects will be + * {@link PoolableObjectFactory#validateObject validated} + * before being returned by the {@link #borrowObject} + * method. If the object fails to validate, + * it will be dropped from the pool, and we will attempt + * to borrow another. + * + * @see #getTestOnBorrow + */ + public synchronized void setTestOnBorrow(boolean testOnBorrow) { + _testOnBorrow = testOnBorrow; + } + + /** + * When true, objects will be + * {@link PoolableObjectFactory#validateObject validated} + * before being returned to the pool within the + * {@link #returnObject}. + * + * @see #setTestOnReturn + */ + public synchronized boolean getTestOnReturn() { + return _testOnReturn; + } + + /** + * When true, objects will be + * {@link PoolableObjectFactory#validateObject validated} + * before being returned to the pool within the + * {@link #returnObject}. + * + * @see #getTestOnReturn + */ + public synchronized void setTestOnReturn(boolean testOnReturn) { + _testOnReturn = testOnReturn; + } + + /** + * Returns the number of milliseconds to sleep between runs of the + * idle object evictor thread. + * When non-positive, no idle object evictor thread will be + * run. + * + * @see #setTimeBetweenEvictionRunsMillis + */ + public synchronized long getTimeBetweenEvictionRunsMillis() { + return _timeBetweenEvictionRunsMillis; + } + + /** + * Sets the number of milliseconds to sleep between runs of the + * idle object evictor thread. + * When non-positive, no idle object evictor thread will be + * run. + * + * @see #getTimeBetweenEvictionRunsMillis + */ + public synchronized void setTimeBetweenEvictionRunsMillis(long timeBetweenEvictionRunsMillis) { + _timeBetweenEvictionRunsMillis = timeBetweenEvictionRunsMillis; + startEvictor(_timeBetweenEvictionRunsMillis); + } + + /** + * Returns the max number of objects to examine during each run of the + * idle object evictor thread (if any). + * + * @see #setNumTestsPerEvictionRun + * @see #setTimeBetweenEvictionRunsMillis + */ + public synchronized int getNumTestsPerEvictionRun() { + return _numTestsPerEvictionRun; + } + + /** + * Sets the max number of objects to examine during each run of the + * idle object evictor thread (if any). + *

+ * When a negative value is supplied, ceil({@link #getNumIdle})/abs({@link #getNumTestsPerEvictionRun}) + * tests will be run. I.e., when the value is -n, roughly one nth of the + * idle objects will be tested per run. + * + * @see #getNumTestsPerEvictionRun + * @see #setTimeBetweenEvictionRunsMillis + */ + public synchronized void setNumTestsPerEvictionRun(int numTestsPerEvictionRun) { + _numTestsPerEvictionRun = numTestsPerEvictionRun; + } + + /** + * Returns the minimum amount of time an object may sit idle in the pool + * before it is eligable for eviction by the idle object evictor + * (if any). + * + * @see #setMinEvictableIdleTimeMillis + * @see #setTimeBetweenEvictionRunsMillis + */ + public synchronized long getMinEvictableIdleTimeMillis() { + return _minEvictableIdleTimeMillis; + } + + /** + * Sets the minimum amount of time an object may sit idle in the pool + * before it is eligable for eviction by the idle object evictor + * (if any). + * When non-positive, no objects will be evicted from the pool + * due to idle time alone. + * + * @see #getMinEvictableIdleTimeMillis + * @see #setTimeBetweenEvictionRunsMillis + */ + public synchronized void setMinEvictableIdleTimeMillis(long minEvictableIdleTimeMillis) { + _minEvictableIdleTimeMillis = minEvictableIdleTimeMillis; + } + + /** + * Returns the minimum amount of time an object may sit idle in the pool + * before it is eligable for eviction by the idle object evictor + * (if any), with the extra condition that at least + * "minIdle" amount of object remain in the pool. + * + * @see #setSoftMinEvictableIdleTimeMillis + */ + public synchronized long getSoftMinEvictableIdleTimeMillis() { + return _softMinEvictableIdleTimeMillis; + } + + /** + * Sets the minimum amount of time an object may sit idle in the pool + * before it is eligable for eviction by the idle object evictor + * (if any), with the extra condition that at least + * "minIdle" amount of object remain in the pool. + * When non-positive, no objects will be evicted from the pool + * due to idle time alone. + * + * @see #getSoftMinEvictableIdleTimeMillis + */ + public synchronized void setSoftMinEvictableIdleTimeMillis(long softMinEvictableIdleTimeMillis) { + _softMinEvictableIdleTimeMillis = softMinEvictableIdleTimeMillis; + } + + /** + * When true, objects will be + * {@link PoolableObjectFactory#validateObject validated} + * by the idle object evictor (if any). If an object + * fails to validate, it will be dropped from the pool. + * + * @see #setTestWhileIdle + * @see #setTimeBetweenEvictionRunsMillis + */ + public synchronized boolean getTestWhileIdle() { + return _testWhileIdle; + } + + /** + * When true, objects will be + * {@link PoolableObjectFactory#validateObject validated} + * by the idle object evictor (if any). If an object + * fails to validate, it will be dropped from the pool. + * + * @see #getTestWhileIdle + * @see #setTimeBetweenEvictionRunsMillis + */ + public synchronized void setTestWhileIdle(boolean testWhileIdle) { + _testWhileIdle = testWhileIdle; + } + + /** + * Sets my configuration. + * @see GenericObjectPool.Config + */ + public synchronized void setConfig(GenericObjectPool.Config conf) { + setMaxIdle(conf.maxIdle); + setMinIdle(conf.minIdle); + setMaxActive(conf.maxActive); + setMaxWait(conf.maxWait); + setWhenExhaustedAction(conf.whenExhaustedAction); + setTestOnBorrow(conf.testOnBorrow); + setTestOnReturn(conf.testOnReturn); + setTestWhileIdle(conf.testWhileIdle); + setNumTestsPerEvictionRun(conf.numTestsPerEvictionRun); + setMinEvictableIdleTimeMillis(conf.minEvictableIdleTimeMillis); + setTimeBetweenEvictionRunsMillis(conf.timeBetweenEvictionRunsMillis); + notifyAll(); + } + + //-- ObjectPool methods ------------------------------------------ + + public synchronized Object borrowObject() throws Exception { + assertOpen(); + long starttime = System.currentTimeMillis(); + for(;;) { + ObjectTimestampPair pair = null; + + // if there are any sleeping, just grab one of those + try { + pair = (ObjectTimestampPair)(_pool.removeFirst()); + } catch(NoSuchElementException e) { + ; /* ignored */ + } + + // otherwise + if(null == pair) { + // check if we can create one + // (note we know that the num sleeping is 0, else we wouldn't be here) + if(_maxActive < 0 || _numActive < _maxActive) { + // allow new object to be created + } else { + // the pool is exhausted + switch(_whenExhaustedAction) { + case WHEN_EXHAUSTED_GROW: + // allow new object to be created + break; + case WHEN_EXHAUSTED_FAIL: + throw new NoSuchElementException("Pool exhausted"); + case WHEN_EXHAUSTED_BLOCK: + try { + if(_maxWait <= 0) { + wait(); + } else { + // this code may be executed again after a notify then continue cycle + // so, need to calculate the amount of time to wait + final long elapsed = (System.currentTimeMillis() - starttime); + final long waitTime = _maxWait - elapsed; + if (waitTime > 0) + { + wait(waitTime); + } + } + } catch(InterruptedException e) { + // ignored + } + if(_maxWait > 0 && ((System.currentTimeMillis() - starttime) >= _maxWait)) { + throw new NoSuchElementException("Timeout waiting for idle object"); + } else { + continue; // keep looping + } + default: + throw new IllegalArgumentException("WhenExhaustedAction property " + _whenExhaustedAction + " not recognized."); + } + } + } + _numActive++; + + // create new object when needed + boolean newlyCreated = false; + if(null == pair) { + try { + Object obj = _factory.makeObject(); + pair = new ObjectTimestampPair(obj); + newlyCreated = true; + } finally { + if (!newlyCreated) { + // object cannot be created + _numActive--; + notifyAll(); + } + } + } + + // activate & validate the object + try { + _factory.activateObject(pair.value); + if(_testOnBorrow && !_factory.validateObject(pair.value)) { + throw new Exception("ValidateObject failed"); + } + return pair.value; + } + catch (Throwable e) { + // object cannot be activated or is invalid + _numActive--; + notifyAll(); + try { + _factory.destroyObject(pair.value); + } + catch (Throwable e2) { + // cannot destroy broken object + } + if(newlyCreated) { + throw new NoSuchElementException("Could not create a validated object, cause: " + e.getMessage()); + } + else { + continue; // keep looping + } + } + } + } + + public synchronized void invalidateObject(Object obj) throws Exception { + assertOpen(); + try { + _factory.destroyObject(obj); + } + finally { + _numActive--; + notifyAll(); // _numActive has changed + } + } + + public synchronized void clear() { + assertOpen(); + for(Iterator it = _pool.iterator(); it.hasNext(); ) { + try { + _factory.destroyObject(((ObjectTimestampPair)(it.next())).value); + } catch(Exception e) { + // ignore error, keep destroying the rest + } + it.remove(); + } + _pool.clear(); + notifyAll(); // num sleeping has changed + } + + public synchronized int getNumActive() { + assertOpen(); + return _numActive; + } + + public synchronized int getNumIdle() { + assertOpen(); + return _pool.size(); + } + + public synchronized void returnObject(Object obj) throws Exception { + assertOpen(); + addObjectToPool(obj, true); + } + + private void addObjectToPool(Object obj, boolean decrementNumActive) throws Exception { + boolean success = true; + if(_testOnReturn && !(_factory.validateObject(obj))) { + success = false; + } else { + try { + _factory.passivateObject(obj); + } catch(Exception e) { + success = false; + } + } + + boolean shouldDestroy = !success; + + if (decrementNumActive) { + _numActive--; + } + if((_maxIdle >= 0) && (_pool.size() >= _maxIdle)) { + shouldDestroy = true; + } else if(success) { + _pool.addLast(new ObjectTimestampPair(obj)); + } + notifyAll(); // _numActive has changed + + if(shouldDestroy) { + try { + _factory.destroyObject(obj); + } catch(Exception e) { + // ignored + } + } + } + + public synchronized void close() throws Exception { + clear(); + _pool = null; + _factory = null; + startEvictor(-1L); + super.close(); + } + + public synchronized void setFactory(PoolableObjectFactory factory) throws IllegalStateException { + assertOpen(); + if(0 < getNumActive()) { + throw new IllegalStateException("Objects are already active"); + } else { + clear(); + _factory = factory; + } + } + + public synchronized void evict() throws Exception { + assertOpen(); + if(!_pool.isEmpty()) { + ListIterator iter; + if (evictLastIndex < 0) { + iter = _pool.listIterator(_pool.size()); + } else { + iter = _pool.listIterator(evictLastIndex); + } + for(int i=0,m=getNumTests();i 0) + && (idleTimeMilis > _minEvictableIdleTimeMillis)) { + removeObject = true; + } else if ((_softMinEvictableIdleTimeMillis > 0) + && (idleTimeMilis > _softMinEvictableIdleTimeMillis) + && (getNumIdle() > getMinIdle())) { + removeObject = true; + } + if(_testWhileIdle && !removeObject) { + boolean active = false; + try { + _factory.activateObject(pair.value); + active = true; + } catch(Exception e) { + removeObject=true; + } + if(active) { + if(!_factory.validateObject(pair.value)) { + removeObject=true; + } else { + try { + _factory.passivateObject(pair.value); + } catch(Exception e) { + removeObject=true; + } + } + } + } + if(removeObject) { + try { + iter.remove(); + _factory.destroyObject(pair.value); + } catch(Exception e) { + // ignored + } + } + } + evictLastIndex = iter.previousIndex(); // resume from here + } // if !empty + } + + /** + * Check to see if we are below our minimum number of objects + * if so enough to bring us back to our minimum. + */ + private void ensureMinIdle() throws Exception { + // this method isn't synchronized so the + // calculateDeficit is done at the beginning + // as a loop limit and a second time inside the loop + // to stop when another thread already returned the + // needed objects + int objectDeficit = calculateDeficit(); + for ( int j = 0 ; j < objectDeficit && calculateDeficit() > 0 ; j++ ) { + addObject(); + } + } + + private synchronized int calculateDeficit() { + int objectDeficit = getMinIdle() - getNumIdle(); + if (_maxActive > 0) { + int growLimit = Math.max(0, getMaxActive() - getNumActive() - getNumIdle()); + objectDeficit = Math.min(objectDeficit, growLimit); + } + return objectDeficit; + } + + /** + * Create an object, and place it into the pool. + * addObject() is useful for "pre-loading" a pool with idle objects. + */ + public synchronized void addObject() throws Exception { + assertOpen(); + Object obj = _factory.makeObject(); + addObjectToPool(obj, false); + } + + //--- non-public methods ---------------------------------------- + + /** + * Start the eviction thread or service, or when + * delay is non-positive, stop it + * if it is already running. + */ + protected synchronized void startEvictor(long delay) { + if(null != _evictor) { + _evictor.cancel(); + _evictor = null; + } + if(delay > 0) { + _evictor = new Evictor(); + EVICTION_TIMER.schedule(_evictor, delay, delay); + } + } + + synchronized String debugInfo() { + StringBuffer buf = new StringBuffer(); + buf.append("Active: ").append(getNumActive()).append("\n"); + buf.append("Idle: ").append(getNumIdle()).append("\n"); + buf.append("Idle Objects:\n"); + Iterator it = _pool.iterator(); + long time = System.currentTimeMillis(); + while(it.hasNext()) { + ObjectTimestampPair pair = (ObjectTimestampPair)(it.next()); + buf.append("\t").append(pair.value).append("\t").append(time - pair.tstamp).append("\n"); + } + return buf.toString(); + } + + private int getNumTests() { + if(_numTestsPerEvictionRun >= 0) { + return Math.min(_numTestsPerEvictionRun, _pool.size()); + } else { + return(int)(Math.ceil((double)_pool.size()/Math.abs((double)_numTestsPerEvictionRun))); + } + } + + //--- inner classes ---------------------------------------------- + + /** + * The idle object evictor {@link TimerTask}. + * @see GenericObjectPool#setTimeBetweenEvictionRunsMillis + */ + private class Evictor extends TimerTask { + public void run() { + try { + evict(); + } catch(Exception e) { + // ignored + } + try { + ensureMinIdle(); + } catch(Exception e) { + // ignored + } + } + } + + /** + * A simple "struct" encapsulating the + * configuration information for a {@link GenericObjectPool}. + * @see GenericObjectPool#GenericObjectPool(org.apache.commons.pool.PoolableObjectFactory,org.apache.commons.pool.impl.GenericObjectPool.Config) + * @see GenericObjectPool#setConfig + */ + public static class Config { + public int maxIdle = GenericObjectPool.DEFAULT_MAX_IDLE; + public int minIdle = GenericObjectPool.DEFAULT_MIN_IDLE; + public int maxActive = GenericObjectPool.DEFAULT_MAX_ACTIVE; + public long maxWait = GenericObjectPool.DEFAULT_MAX_WAIT; + public byte whenExhaustedAction = GenericObjectPool.DEFAULT_WHEN_EXHAUSTED_ACTION; + public boolean testOnBorrow = GenericObjectPool.DEFAULT_TEST_ON_BORROW; + public boolean testOnReturn = GenericObjectPool.DEFAULT_TEST_ON_RETURN; + public boolean testWhileIdle = GenericObjectPool.DEFAULT_TEST_WHILE_IDLE; + public long timeBetweenEvictionRunsMillis = GenericObjectPool.DEFAULT_TIME_BETWEEN_EVICTION_RUNS_MILLIS; + public int numTestsPerEvictionRun = GenericObjectPool.DEFAULT_NUM_TESTS_PER_EVICTION_RUN; + public long minEvictableIdleTimeMillis = GenericObjectPool.DEFAULT_MIN_EVICTABLE_IDLE_TIME_MILLIS; + public long softMinEvictableIdleTimeMillis = GenericObjectPool.DEFAULT_MIN_EVICTABLE_IDLE_TIME_MILLIS; + } + + //--- private attributes --------------------------------------- + + /** + * The cap on the number of idle instances in the pool. + * @see #setMaxIdle + * @see #getMaxIdle + */ + private int _maxIdle = DEFAULT_MAX_IDLE; + + /** + * The cap on the minimum number of idle instances in the pool. + * @see #setMinIdle + * @see #getMinIdle + */ + private int _minIdle = DEFAULT_MIN_IDLE; + + /** + * The cap on the total number of active instances from the pool. + * @see #setMaxActive + * @see #getMaxActive + */ + protected int _maxActive = DEFAULT_MAX_ACTIVE; + + /** + * The maximum amount of time (in millis) the + * {@link #borrowObject} method should block before throwing + * an exception when the pool is exhausted and the + * {@link #getWhenExhaustedAction "when exhausted" action} is + * {@link #WHEN_EXHAUSTED_BLOCK}. + * + * When less than 0, the {@link #borrowObject} method + * may block indefinitely. + * + * @see #setMaxWait + * @see #getMaxWait + * @see #WHEN_EXHAUSTED_BLOCK + * @see #setWhenExhaustedAction + * @see #getWhenExhaustedAction + */ + protected long _maxWait = DEFAULT_MAX_WAIT; + + /** + * The action to take when the {@link #borrowObject} method + * is invoked when the pool is exhausted (the maximum number + * of "active" objects has been reached). + * + * @see #WHEN_EXHAUSTED_BLOCK + * @see #WHEN_EXHAUSTED_FAIL + * @see #WHEN_EXHAUSTED_GROW + * @see #DEFAULT_WHEN_EXHAUSTED_ACTION + * @see #setWhenExhaustedAction + * @see #getWhenExhaustedAction + */ + protected byte _whenExhaustedAction = DEFAULT_WHEN_EXHAUSTED_ACTION; + + /** + * When true, objects will be + * {@link PoolableObjectFactory#validateObject validated} + * before being returned by the {@link #borrowObject} + * method. If the object fails to validate, + * it will be dropped from the pool, and we will attempt + * to borrow another. + * + * @see #setTestOnBorrow + * @see #getTestOnBorrow + */ + protected boolean _testOnBorrow = DEFAULT_TEST_ON_BORROW; + + /** + * When true, objects will be + * {@link PoolableObjectFactory#validateObject validated} + * before being returned to the pool within the + * {@link #returnObject}. + * + * @see #getTestOnReturn + * @see #setTestOnReturn + */ + private boolean _testOnReturn = DEFAULT_TEST_ON_RETURN; + + /** + * When true, objects will be + * {@link PoolableObjectFactory#validateObject validated} + * by the idle object evictor (if any). If an object + * fails to validate, it will be dropped from the pool. + * + * @see #setTestWhileIdle + * @see #getTestWhileIdle + * @see #getTimeBetweenEvictionRunsMillis + * @see #setTimeBetweenEvictionRunsMillis + */ + private boolean _testWhileIdle = DEFAULT_TEST_WHILE_IDLE; + + /** + * The number of milliseconds to sleep between runs of the + * idle object evictor thread. + * When non-positive, no idle object evictor thread will be + * run. + * + * @see #setTimeBetweenEvictionRunsMillis + * @see #getTimeBetweenEvictionRunsMillis + */ + private long _timeBetweenEvictionRunsMillis = DEFAULT_TIME_BETWEEN_EVICTION_RUNS_MILLIS; + + /** + * The max number of objects to examine during each run of the + * idle object evictor thread (if any). + *

+ * When a negative value is supplied, ceil({@link #getNumIdle})/abs({@link #getNumTestsPerEvictionRun}) + * tests will be run. I.e., when the value is -n, roughly one nth of the + * idle objects will be tested per run. + * + * @see #setNumTestsPerEvictionRun + * @see #getNumTestsPerEvictionRun + * @see #getTimeBetweenEvictionRunsMillis + * @see #setTimeBetweenEvictionRunsMillis + */ + private int _numTestsPerEvictionRun = DEFAULT_NUM_TESTS_PER_EVICTION_RUN; + + /** + * The minimum amount of time an object may sit idle in the pool + * before it is eligable for eviction by the idle object evictor + * (if any). + * When non-positive, no objects will be evicted from the pool + * due to idle time alone. + * + * @see #setMinEvictableIdleTimeMillis + * @see #getMinEvictableIdleTimeMillis + * @see #getTimeBetweenEvictionRunsMillis + * @see #setTimeBetweenEvictionRunsMillis + */ + private long _minEvictableIdleTimeMillis = DEFAULT_MIN_EVICTABLE_IDLE_TIME_MILLIS; + + /** + * The minimum amount of time an object may sit idle in the pool + * before it is eligable for eviction by the idle object evictor + * (if any), with the extra condition that at least + * "minIdle" amount of object remain in the pool. + * When non-positive, no objects will be evicted from the pool + * due to idle time alone. + * + * @see #setSoftMinEvictableIdleTimeMillis + * @see #getSoftMinEvictableIdleTimeMillis + */ + private long _softMinEvictableIdleTimeMillis = DEFAULT_SOFT_MIN_EVICTABLE_IDLE_TIME_MILLIS; + + /** My pool. */ + protected LinkedList _pool = null; + + /** My {@link PoolableObjectFactory}. */ + protected PoolableObjectFactory _factory = null; + + /** + * The number of objects {@link #borrowObject} borrowed + * from the pool, but not yet returned. + */ + protected int _numActive = 0; + + /** + * My idle object eviction {@link TimerTask}, if any. + */ + private Evictor _evictor = null; + + /** + * Position in the _pool where the _evictor last stopped. + */ + private int evictLastIndex = -1; +} diff --git a/commons/src/main/java/org/archive/bdb/BdbModule.java b/commons/src/main/java/org/archive/bdb/BdbModule.java new file mode 100644 index 00000000..2655b6a0 --- /dev/null +++ b/commons/src/main/java/org/archive/bdb/BdbModule.java @@ -0,0 +1,637 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.bdb; + +import java.io.Closeable; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.PrintWriter; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.io.FileUtils; +import org.archive.checkpointing.CheckpointRecovery; +import org.archive.checkpointing.Checkpointable; +import org.archive.checkpointing.RecoverAction; +import org.archive.spring.ConfigPath; +import org.archive.util.CachedBdbMap; +import org.archive.util.bdbje.EnhancedEnvironment; +import org.springframework.context.Lifecycle; + +import com.sleepycat.bind.serial.StoredClassCatalog; +import com.sleepycat.je.CheckpointConfig; +import com.sleepycat.je.Database; +import com.sleepycat.je.DatabaseConfig; +import com.sleepycat.je.DatabaseException; +import com.sleepycat.je.DatabaseNotFoundException; +import com.sleepycat.je.DbInternal; +import com.sleepycat.je.EnvironmentConfig; +import com.sleepycat.je.SecondaryConfig; +import com.sleepycat.je.SecondaryDatabase; +import com.sleepycat.je.SecondaryKeyCreator; +import com.sleepycat.je.dbi.EnvironmentImpl; +import com.sleepycat.je.utilint.DbLsn; + +/** + * Utility module for managing a shared BerkeleyDB-JE environment + * + * @contributor pjack + * @contributor gojomo + */ +public class BdbModule implements Lifecycle, Checkpointable, +Serializable, Closeable { + private static final long serialVersionUID = 1L; + final private static Logger LOGGER = + Logger.getLogger(BdbModule.class.getName()); + + + private static class DatabasePlusConfig implements Serializable { + private static final long serialVersionUID = 1L; + public transient Database database; + public String name; + public String primaryName; + public BdbConfig config; + } + + + /** + * Configuration object for databases. Needed because + * {@link DatabaseConfig} is not serializable. Also it prevents invalid + * configurations. (All databases opened through this module must be + * deferred-write, because otherwise they can't sync(), and you can't + * run a checkpoint without doing sync() first.) + * + * @author pjack + * + */ + public static class BdbConfig implements Serializable { + private static final long serialVersionUID = 1L; + + boolean allowCreate; + boolean sortedDuplicates; + boolean transactional; + + + public BdbConfig() { + } + + + public boolean isAllowCreate() { + return allowCreate; + } + + + public void setAllowCreate(boolean allowCreate) { + this.allowCreate = allowCreate; + } + + + public boolean getSortedDuplicates() { + return sortedDuplicates; + } + + + public void setSortedDuplicates(boolean sortedDuplicates) { + this.sortedDuplicates = sortedDuplicates; + } + + public DatabaseConfig toDatabaseConfig() { + DatabaseConfig result = new DatabaseConfig(); + result.setDeferredWrite(true); + result.setTransactional(transactional); + result.setAllowCreate(allowCreate); + result.setSortedDuplicates(sortedDuplicates); + return result; + } + + + public boolean isTransactional() { + return transactional; + } + + + public void setTransactional(boolean transactional) { + this.transactional = transactional; + } + } + + + public static class SecondaryBdbConfig extends BdbConfig { + private static final long serialVersionUID = 1L; + + private SecondaryKeyCreator keyCreator; + + public SecondaryBdbConfig() { + } + + public SecondaryKeyCreator getKeyCreator() { + return keyCreator; + } + + public void setKeyCreator(SecondaryKeyCreator keyCreator) { + this.keyCreator = keyCreator; + } + + public SecondaryConfig toSecondaryConfig() { + SecondaryConfig result = new SecondaryConfig(); + result.setDeferredWrite(true); + result.setTransactional(transactional); + result.setAllowCreate(allowCreate); + result.setSortedDuplicates(sortedDuplicates); + result.setKeyCreator(keyCreator); + return result; + } + + } + + protected ConfigPath dir = new ConfigPath("state subdirectory","state"); + public ConfigPath getDir() { + return dir; + } + public void setDir(ConfigPath dir) { + this.dir = dir; + } + + int cachePercent = 60; + public int getCachePercent() { + return cachePercent; + } + public void setCachePercent(int cachePercent) { + this.cachePercent = cachePercent; + } + + boolean useSharedCache = true; + public boolean getUseSharedCache() { + return useSharedCache; + } + public void setUseSharedCache(boolean useSharedCache) { + this.useSharedCache = useSharedCache; + } + + boolean checkpointCopyLogs = true; + public boolean getCheckpointCopyLogs() { + return checkpointCopyLogs; + } + public void setCheckpointCopyLogs(boolean checkpointCopyLogs) { + this.checkpointCopyLogs = checkpointCopyLogs; + } + + private transient EnhancedEnvironment bdbEnvironment; + + private transient StoredClassCatalog classCatalog; + + @SuppressWarnings("unchecked") + private Map bigMaps = + new ConcurrentHashMap(); + + private Map databases = + new ConcurrentHashMap(); + + private transient Thread shutdownHook; + + public BdbModule() { + } + + + public void start() { + if (isRunning()) { + return; + } + try { + setUp(getDir().getFile(), getCachePercent(), true, getUseSharedCache()); + } catch (DatabaseException e) { + throw new IllegalStateException(e); + } + shutdownHook = new BdbShutdownHook(this); + Runtime.getRuntime().addShutdownHook(shutdownHook); + } + + public boolean isRunning() { + return shutdownHook!=null; + } + + public void stop() { + if (!isRunning()) { + return; + } + close(); + } + + private void setUp(File f, int cachePercent, boolean create, boolean sharedCache) + throws DatabaseException { + EnvironmentConfig config = new EnvironmentConfig(); + config.setAllowCreate(create); + config.setLockTimeout(5000000); + config.setCachePercent(cachePercent); + config.setSharedCache(sharedCache); + + f.mkdirs(); + this.bdbEnvironment = new EnhancedEnvironment(f, config); + + this.classCatalog = this.bdbEnvironment.getClassCatalog(); + } + + + public void closeDatabase(Database db) { + try { + closeDatabase(db.getDatabaseName()); + } catch (DatabaseException e) { + LOGGER.log(Level.SEVERE, "Error getting db name", e); + } + } + + public void closeDatabase(String name) { + DatabasePlusConfig dpc = databases.remove(name); + if (dpc == null) { + throw new IllegalStateException("No such database: " + name); + } + Database db = dpc.database; + try { + db.sync(); + db.close(); + } catch (DatabaseException e) { + LOGGER.log(Level.SEVERE, "Error closing db " + name, e); + } + } + + + public Database openDatabase(String name, BdbConfig config, + boolean recycle) + throws DatabaseException { + if (databases.containsKey(name)) { + throw new IllegalStateException("Database already exists: " +name); + } + if (!recycle) { + try { + bdbEnvironment.truncateDatabase(null, name, false); + } catch (DatabaseNotFoundException e) { + // Ignored + } + } + DatabasePlusConfig dpc = new DatabasePlusConfig(); + dpc.database = bdbEnvironment.openDatabase(null, name, config.toDatabaseConfig()); + dpc.name = name; + dpc.config = config; + databases.put(name, dpc); + return dpc.database; + } + + + public SecondaryDatabase openSecondaryDatabase(String name, Database db, + SecondaryBdbConfig config) throws DatabaseException { + if (databases.containsKey(name)) { + throw new IllegalStateException("Database already exists: " +name); + } + SecondaryDatabase result = bdbEnvironment.openSecondaryDatabase(null, + name, db, config.toSecondaryConfig()); + DatabasePlusConfig dpc = new DatabasePlusConfig(); + dpc.database = result; + dpc.name = name; + dpc.primaryName = db.getDatabaseName(); + dpc.config = config; + databases.put(name, dpc); + return result; + } + + public StoredClassCatalog getClassCatalog() { + return classCatalog; + } + + + public Map getBigMap(String dbName, boolean recycle, + Class key, Class value) + throws DatabaseException { + @SuppressWarnings("unchecked") + CachedBdbMap r = bigMaps.get(dbName); + if (r != null) { + return r; + } + + if (!recycle) { + try { + bdbEnvironment.truncateDatabase(null, dbName, false); + } catch (DatabaseNotFoundException e) { + // ignored + } + } + + r = new CachedBdbMap(dbName); + + r.initialize(bdbEnvironment, key, value, classCatalog); + bigMaps.put(dbName, r); + return r; + } + + + private void writeObject(ObjectOutputStream out) throws IOException { + out.defaultWriteObject(); + } + + + + @SuppressWarnings("unchecked") + private void readObject(ObjectInputStream in) + throws IOException, ClassNotFoundException { + in.defaultReadObject(); + if (in instanceof CheckpointRecovery) { +// CheckpointRecovery cr = (CheckpointRecovery)in; +// path = cr.translatePath(path); +// cr.setState(this, DIR, path); + } + try { + setUp(getDir().getFile(), getCachePercent(), false, getUseSharedCache()); + for (CachedBdbMap map: bigMaps.values()) { + map.initialize( + this.bdbEnvironment, + map.getKeyClass(), + map.getValueClass(), + this.classCatalog); + } + for (DatabasePlusConfig dpc: databases.values()) { + if (!(dpc.config instanceof SecondaryBdbConfig)) { + dpc.database = bdbEnvironment.openDatabase(null, + dpc.name, dpc.config.toDatabaseConfig()); + } + } + for (DatabasePlusConfig dpc: databases.values()) { + if (dpc.config instanceof SecondaryBdbConfig) { + SecondaryBdbConfig conf = (SecondaryBdbConfig)dpc.config; + Database primary = databases.get(dpc.primaryName).database; + dpc.database = bdbEnvironment.openSecondaryDatabase(null, + dpc.name, primary, conf.toSecondaryConfig()); + } + } + } catch (DatabaseException e) { + IOException io = new IOException(); + io.initCause(e); + throw io; + } + this.shutdownHook = new BdbShutdownHook(this); + Runtime.getRuntime().addShutdownHook(shutdownHook); + } + + + @SuppressWarnings("unchecked") + public void checkpoint(File dir, List actions) + throws IOException { + if (checkpointCopyLogs) { + actions.add(new BdbRecover(getDir().getFile().getAbsolutePath())); + } + // First sync bigMaps + for (Map.Entry me: bigMaps.entrySet()) { + me.getValue().sync(); + } + + EnvironmentConfig envConfig; + try { + // sync all databases + for (DatabasePlusConfig dbc: databases.values()) { + dbc.database.sync(); + } + envConfig = bdbEnvironment.getConfig(); + } catch (DatabaseException e) { + IOException io = new IOException(); + io.initCause(e); + throw io; + } + + final List bkgrdThreads = Arrays.asList(new String [] + {"je.env.runCheckpointer", "je.env.runCleaner", + "je.env.runINCompressor"}); + try { + // Disable background threads + setBdbjeBkgrdThreads(envConfig, bkgrdThreads, "false"); + // Do a force checkpoint. Thats what a sync does (i.e. doSync). + CheckpointConfig chkptConfig = new CheckpointConfig(); + chkptConfig.setForce(true); + + // Mark Hayes of sleepycat says: + // "The default for this property is false, which gives the current + // behavior (allow deltas). If this property is true, deltas are + // prohibited -- full versions of internal nodes are always logged + // during the checkpoint. When a full version of an internal node + // is logged during a checkpoint, recovery does not need to process + // it at all. It is only fetched if needed by the application, + // during normal DB operations after recovery. When a delta of an + // internal node is logged during a checkpoint, recovery must + // process it by fetching the full version of the node from earlier + // in the log, and then applying the delta to it. This can be + // pretty slow, since it is potentially a large amount of + // random I/O." + chkptConfig.setMinimizeRecoveryTime(true); + bdbEnvironment.checkpoint(chkptConfig); + LOGGER.fine("Finished bdb checkpoint."); + + // From the sleepycat folks: A trick for flipping db logs. + EnvironmentImpl envImpl = + DbInternal.envGetEnvironmentImpl(bdbEnvironment); + long firstFileInNextSet = + DbLsn.getFileNumber(envImpl.forceLogFileFlip()); + // So the last file in the checkpoint is firstFileInNextSet - 1. + // Write manifest of all log files into the bdb directory. + final String lastBdbCheckpointLog = + getBdbLogFileName(firstFileInNextSet - 1); + processBdbLogs(dir, lastBdbCheckpointLog); + LOGGER.fine("Finished processing bdb log files."); + } catch (DatabaseException e) { + IOException io = new IOException(); + io.initCause(e); + throw io; + } finally { + // Restore background threads. + setBdbjeBkgrdThreads(envConfig, bkgrdThreads, "true"); + } + } + + + private void processBdbLogs(final File checkpointDir, + final String lastBdbCheckpointLog) throws IOException { + File bdbDir = getBdbSubDirectory(checkpointDir); + if (!bdbDir.exists()) { + bdbDir.mkdir(); + } + PrintWriter pw = new PrintWriter(new FileOutputStream(new File( + checkpointDir, "bdbje-logs-manifest.txt"))); + try { + // Don't copy any beyond the last bdb log file (bdbje can keep + // writing logs after checkpoint). + boolean pastLastLogFile = false; + Set srcFilenames = null; + do { + FilenameFilter filter = new FilenameFilter() { + public boolean accept(File dir, String name) { + return name != null + && name.toLowerCase().endsWith(".jdb"); + } + }; + + srcFilenames = + new HashSet(Arrays.asList(getDir().getFile().list(filter))); + List tgtFilenames = Arrays.asList(bdbDir.list(filter)); + if (tgtFilenames != null && tgtFilenames.size() > 0) { + srcFilenames.removeAll(tgtFilenames); + } + if (srcFilenames.size() > 0) { + // Sort files. + srcFilenames = new TreeSet(srcFilenames); + int count = 0; + for (final Iterator i = srcFilenames.iterator(); + i.hasNext() && !pastLastLogFile;) { + String name = (String) i.next(); + if (this.checkpointCopyLogs) { + FileUtils.copyDirectory(new File(getDir().getFile(), name), + new File(bdbDir, name)); + } + pw.println(name); + if (name.equals(lastBdbCheckpointLog)) { + // We're done. + pastLastLogFile = true; + } + count++; + } + if (LOGGER.isLoggable(Level.FINE)) { + LOGGER.fine("Copied " + count); + } + } + } while (!pastLastLogFile && srcFilenames != null && + srcFilenames.size() > 0); + } finally { + pw.close(); + } + } + + + + private void setBdbjeBkgrdThreads(final EnvironmentConfig config, + final List threads, final String setting) { + for (final Iterator i = threads.iterator(); i.hasNext();) { + config.setConfigParam((String)i.next(), setting); + } + } + + + private String getBdbLogFileName(final long index) { + String lastBdbLogFileHex = Long.toHexString(index); + StringBuffer buffer = new StringBuffer(); + for (int i = 0; i < (8 - lastBdbLogFileHex.length()); i++) { + buffer.append('0'); + } + buffer.append(lastBdbLogFileHex); + buffer.append(".jdb"); + return buffer.toString(); + } + + + public void close() { + close2(); + Runtime.getRuntime().removeShutdownHook(shutdownHook); + shutdownHook = null; + } + + @SuppressWarnings("unchecked") + void close2() { + if (classCatalog == null) { + return; + } + for (Map.Entry me: bigMaps.entrySet()) try { + me.getValue().close(); + } catch (Exception e) { + LOGGER.log(Level.SEVERE, "Error closing bigMap " + me.getKey(), e); + } + + List dbNames = new ArrayList(databases.keySet()); + for (String dbName: dbNames) try { + closeDatabase(dbName); + } catch (Exception e) { + LOGGER.log(Level.SEVERE, "Error closing db " + dbName, e); + } + + try { + this.bdbEnvironment.sync(); + this.bdbEnvironment.close(); + } catch (Exception e) { + LOGGER.log(Level.SEVERE, "Error closing environment.", e); + } + } + + + private static File getBdbSubDirectory(File checkpointDir) { + return new File(checkpointDir, "bdbje-logs"); + } + + + public Database getDatabase(String name) { + DatabasePlusConfig dpc = databases.get(name); + if (dpc == null) { + return null; + } + return dpc.database; + } + + + private static class BdbRecover implements RecoverAction { + + private static final long serialVersionUID = 1L; + + private String path; + + public BdbRecover(String path) { + this.path = path; + } + + public void recoverFrom(File checkpointDir, + CheckpointRecovery recovery) throws Exception { + File bdbDir = getBdbSubDirectory(checkpointDir); + path = recovery.translatePath(path); + FileUtils.copyDirectory(bdbDir, new File(path)); + } + + } + + + private static class BdbShutdownHook extends Thread { + + final private BdbModule bdb; + + + public BdbShutdownHook(BdbModule bdb) { + this.bdb = bdb; + } + + public void run() { + this.bdb.close2(); + } + + } +} diff --git a/commons/src/main/java/org/archive/checkpointing/CheckpointInputStream.java b/commons/src/main/java/org/archive/checkpointing/CheckpointInputStream.java new file mode 100644 index 00000000..371a9113 --- /dev/null +++ b/commons/src/main/java/org/archive/checkpointing/CheckpointInputStream.java @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2007 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * CheckpointInputStream.java + * + * Created on Mar 6, 2007 + * + * $Id:$ + */ + +package org.archive.checkpointing; + +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; +import java.net.URI; + + +/** + * Object input stream that provides information useful during checkpoint + * recovery. + * + * @author pjack + */ +public class CheckpointInputStream extends ObjectInputStream +implements CheckpointRecovery { + + + final private CheckpointRecovery recovery; + + + public CheckpointInputStream(InputStream input, + CheckpointRecovery recovery) throws IOException { + super(input); + this.recovery = recovery; + } + + + public String getRecoveredJobName() { + return recovery.getRecoveredJobName(); + } + +// public void setState(Object module, Key key, T value) { +// recovery.setState(module, key, value); +// } + + + public String translatePath(String path) { + return recovery.translatePath(path); + } + + + public URI translateURI(URI uri) { + return recovery.translateURI(uri); + } + + +// public void apply(SingleSheet global) { +// throw new UnsupportedOperationException(); +// } +} diff --git a/commons/src/main/java/org/archive/checkpointing/CheckpointRecovery.java b/commons/src/main/java/org/archive/checkpointing/CheckpointRecovery.java new file mode 100644 index 00000000..8a78c31f --- /dev/null +++ b/commons/src/main/java/org/archive/checkpointing/CheckpointRecovery.java @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2007 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * CheckpointStream.java + * + * Created on Mar 8, 2007 + * + * $Id:$ + */ + +package org.archive.checkpointing; + +import java.net.URI; + +/** + * The information about a checkpoint recovery. + * + * @author pjack + */ +public interface CheckpointRecovery { + + String getRecoveredJobName(); + + String translatePath(String path); + + URI translateURI(URI uri); + +// void setState(Object module, Key key, T value); +// +// void apply(SingleSheet global); +} diff --git a/commons/src/main/java/org/archive/checkpointing/Checkpointable.java b/commons/src/main/java/org/archive/checkpointing/Checkpointable.java new file mode 100644 index 00000000..a615609d --- /dev/null +++ b/commons/src/main/java/org/archive/checkpointing/Checkpointable.java @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2007 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Checkpointable.java + * + * Created on Mar 1, 2007 + * + * $Id:$ + */ + +package org.archive.checkpointing; + +import java.io.File; +import java.io.IOException; +import java.util.List; + + +/** + * @author pjack + * + */ +public interface Checkpointable { + + + void checkpoint(File dir, List actions) throws IOException; + + +} diff --git a/commons/src/main/java/org/archive/checkpointing/Checkpointer.java b/commons/src/main/java/org/archive/checkpointing/Checkpointer.java new file mode 100644 index 00000000..4e2b999c --- /dev/null +++ b/commons/src/main/java/org/archive/checkpointing/Checkpointer.java @@ -0,0 +1,120 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.checkpointing; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.util.ArrayList; +import java.util.List; + +import org.archive.util.IoUtils; + +/** + * Executes checkpoints and recovers. + * + * @author pjack + */ +public class Checkpointer { + + final public static String ACTIONS_FILE = "actions.serialized"; + + final public static String OBJECT_GRAPH_FILE = "object_graph.serialized"; + + private Checkpointer() { + } + + + public static void checkpoint(/*SheetManager*/Object mgr, File dir) + throws IOException { + List actions = new ArrayList(); +// for (Checkpointable c: mgr.getCheckpointables()) { +// c.checkpoint(dir, actions); +// } + + writeObject(new File(dir, ACTIONS_FILE), actions); + writeObject(new File(dir, OBJECT_GRAPH_FILE), mgr); + } + + + private static void writeObject(File f, Object o) + throws IOException { + ObjectOutputStream oout = null; + try { + oout = new ObjectOutputStream(new FileOutputStream(f)); + oout.writeObject(o); + } finally { + IoUtils.close(oout); + } + + } + + + @SuppressWarnings("unused") + private static List readActions(File dir) + throws IOException { + File actionsFile = new File(dir, ACTIONS_FILE); + ObjectInputStream oinp = null; + try { + oinp = new ObjectInputStream( + new FileInputStream(actionsFile)); + @SuppressWarnings("unchecked") + List actions = (List)oinp.readObject(); + return actions; + } catch (ClassNotFoundException e) { + IOException io = new IOException(); + io.initCause(e); + throw io; + } finally { + IoUtils.close(oinp); + } + } + +// public static SheetManager recover(File dir, CheckpointRecovery recovery) +// throws IOException { +// List actions = readActions(dir); +// for (RecoverAction action: actions) try { +// action.recoverFrom(dir, recovery); +// } catch (Exception e) { +// IOException io = new IOException(); +// io.initCause(e); +// throw io; +// } +// +// CheckpointInputStream cinp = null; +// try { +// File f = new File(dir, OBJECT_GRAPH_FILE); +// cinp = new CheckpointInputStream(new FileInputStream(f), recovery); +// SheetManager mgr = (SheetManager)cinp.readObject(); +// recovery.apply(mgr.getGlobalSheet()); +// return mgr; +// } catch (ClassNotFoundException e) { +// IOException io = new IOException(); +// io.initCause(e); +// throw io; +// }finally { +// IoUtils.close(cinp); +// } +// } + +} diff --git a/commons/src/main/java/org/archive/checkpointing/DefaultCheckpointRecovery.java b/commons/src/main/java/org/archive/checkpointing/DefaultCheckpointRecovery.java new file mode 100644 index 00000000..a92e2b25 --- /dev/null +++ b/commons/src/main/java/org/archive/checkpointing/DefaultCheckpointRecovery.java @@ -0,0 +1,109 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.checkpointing; + +import java.net.URI; +import java.util.HashMap; +import java.util.Map; + +/** + * Default implementation. + * + * @author pjack + * + */ +public class DefaultCheckpointRecovery implements CheckpointRecovery { + + +// final private Map> newSettings = +// new IdentityHashMap>(); + + final private Map uriTranslations = new HashMap(); + + final private Map fileTranslations = + new HashMap(); + + final private String name; + + public DefaultCheckpointRecovery(String name) { + this.name = name; + } + + public String getRecoveredJobName() { + return name; + } + + public Map getFileTranslations() { + return fileTranslations; + } + + public Map getURITranslations() { + return uriTranslations; + } + + +// public void setState(Object module, Key key, T value) { +// Map map = newSettings.get(module); +// if (map == null) { +// map = new HashMap(); +// newSettings.put(module, map); +// } +// +// map.put(key, value); +// } + + + public String translatePath(String path) { + Map.Entry match = null; + for (Map.Entry me: fileTranslations.entrySet()) { + if (path.startsWith(me.getKey())) { + if ((match == null) + || (match.getKey().length() < me.getKey().length())) { + match = me; + } + } + } + + if (match == null) { + return path; + } + + int size = match.getKey().length(); + return match.getValue() + path.substring(size); + } + + + public URI translateURI(URI uri) { + URI r = uriTranslations.get(uri); + return r == null ? uri : r; + } + + +// public void apply(SingleSheet global) { +// for (Map.Entry> mod: newSettings.entrySet()) { +// Object module = mod.getKey(); +// for (Map.Entry me: mod.getValue().entrySet()) { +// @SuppressWarnings("unchecked") +// Key k = me.getKey(); +// global.set(module, k, me.getValue()); +// } +// } +// } +} diff --git a/commons/src/main/java/org/archive/checkpointing/RecoverAction.java b/commons/src/main/java/org/archive/checkpointing/RecoverAction.java new file mode 100644 index 00000000..7ae7a68a --- /dev/null +++ b/commons/src/main/java/org/archive/checkpointing/RecoverAction.java @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2007 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * RecoverAction.java + * + * Created on Mar 2, 2007 + * + * $Id:$ + */ + +package org.archive.checkpointing; + +import java.io.File; +import java.io.Serializable; + + + +/** + * An action to be taken before a checkpoint's object graph is deserialized. + * + * @author pjack + */ +public interface RecoverAction extends Serializable { + + + void recoverFrom(File checkpointDir, CheckpointRecovery recovery) + throws Exception; + +} diff --git a/commons/src/main/java/org/archive/httpclient/ConfigurableX509TrustManager.java b/commons/src/main/java/org/archive/httpclient/ConfigurableX509TrustManager.java new file mode 100644 index 00000000..35ae2b06 --- /dev/null +++ b/commons/src/main/java/org/archive/httpclient/ConfigurableX509TrustManager.java @@ -0,0 +1,192 @@ +/* ConfigurableX509TrustManager + * + * Created on Feb 18, 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.httpclient; + +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.util.logging.Logger; + +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509TrustManager; + +/** + * A configurable trust manager built on X509TrustManager. + * + * If set to 'open' trust, the default, will get us into sites for whom we do + * not have the CA or any of intermediary CAs that go to make up the cert chain + * of trust. Will also get us past selfsigned and expired certs. 'loose' + * trust will get us into sites w/ valid certs even if they are just + * selfsigned. 'normal' is any valid cert not including selfsigned. 'strict' + * means cert must be valid and the cert DN must match server name. + * + *

Based on pointers in + * SSL + * Guide, + * and readings done in JSSE + * Guide. + * + *

TODO: Move to an ssl subpackage when we have other classes other than + * just this one. + * + * @author stack + * @version $Id$ + */ +public class ConfigurableX509TrustManager implements X509TrustManager +{ + /** + * Logging instance. + */ + protected static Logger logger = Logger.getLogger( + "org.archive.httpclient.ConfigurableX509TrustManager"); + + public static enum TrustLevel { + /** + * Trust anything given us. + * + * Default setting. + * + *

See + * e502. Disabling Certificate Validation in an HTTPS Connection from + * the java almanac for how to trust all. + */ + OPEN, + + /** + * Trust any valid cert including self-signed certificates. + */ + LOOSE, + + /** + * Normal jsse behavior. + * + * Seemingly any certificate that supplies valid chain of trust. + */ + NORMAL, + + /** + * Strict trust. + * + * Ensure server has same name as cert DN. + */ + STRICT, + } + + /** + * Default setting for trust level. + */ + public final static TrustLevel DEFAULT = TrustLevel.OPEN; + + /** + * Trust level. + */ + private TrustLevel trustLevel = DEFAULT; + + + /** + * An instance of the SUNX509TrustManager that we adapt variously + * depending upon passed configuration. + * + * We have it do all the work we don't want to. + */ + private X509TrustManager standardTrustManager = null; + + + public ConfigurableX509TrustManager() + throws NoSuchAlgorithmException, KeyStoreException { + this(DEFAULT); + } + + /** + * Constructor. + * + * @param level Level of trust to effect. + * + * @throws NoSuchAlgorithmException + * @throws KeyStoreException + */ + public ConfigurableX509TrustManager(TrustLevel level) + throws NoSuchAlgorithmException, KeyStoreException { + super(); + TrustManagerFactory factory = TrustManagerFactory. + getInstance(TrustManagerFactory.getDefaultAlgorithm()); + + // Pass in a null (Trust) KeyStore. Null says use the 'default' + // 'trust' keystore (KeyStore class is used to hold keys and to hold + // 'trusts' (certs)). See 'X509TrustManager Interface' in this doc: + // http://java.sun.com + // /j2se/1.4.2/docs/guide/security/jsse/JSSERefGuide.html#Introduction + factory.init((KeyStore)null); + TrustManager[] trustmanagers = factory.getTrustManagers(); + if (trustmanagers.length == 0) { + throw new NoSuchAlgorithmException(TrustManagerFactory. + getDefaultAlgorithm() + " trust manager not supported"); + } + this.standardTrustManager = (X509TrustManager)trustmanagers[0]; + + this.trustLevel = level; + } + + public void checkClientTrusted(X509Certificate[] certificates, String type) + throws CertificateException { + if (this.trustLevel.equals(TrustLevel.OPEN)) { + return; + } + + this.standardTrustManager.checkClientTrusted(certificates, type); + } + + public void checkServerTrusted(X509Certificate[] certificates, String type) + throws CertificateException { + if (this.trustLevel.equals(TrustLevel.OPEN)) { + return; + } + + try { + this.standardTrustManager.checkServerTrusted(certificates, type); + if (this.trustLevel.equals(TrustLevel.STRICT)) { + logger.severe(TrustLevel.STRICT + " not implemented."); + } + } catch (CertificateException e) { + if (this.trustLevel.equals(TrustLevel.LOOSE) && + certificates != null && certificates.length == 1) + { + // If only one cert and its valid and it caused a + // CertificateException, assume its selfsigned. + X509Certificate certificate = certificates[0]; + certificate.checkValidity(); + } else { + // If we got to here, then we're probably NORMAL. Rethrow. + throw e; + } + } + } + + public X509Certificate[] getAcceptedIssuers() { + return this.standardTrustManager.getAcceptedIssuers(); + } +} diff --git a/commons/src/main/java/org/archive/httpclient/HttpRecorderGetMethod.java b/commons/src/main/java/org/archive/httpclient/HttpRecorderGetMethod.java new file mode 100644 index 00000000..090ab367 --- /dev/null +++ b/commons/src/main/java/org/archive/httpclient/HttpRecorderGetMethod.java @@ -0,0 +1,124 @@ +/* HttpRecorderGetMethod + * + * Created on Feb 24, 2004 + * + * Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.httpclient; + +import java.io.IOException; +import java.util.logging.Logger; + +import org.apache.commons.httpclient.HttpConnection; +import org.apache.commons.httpclient.HttpException; +import org.apache.commons.httpclient.HttpState; +import org.apache.commons.httpclient.methods.GetMethod; +import org.archive.util.Recorder; + + +/** + * Override of GetMethod that marks the passed HttpRecorder w/ the transition + * from HTTP head to body and that forces a close on the http connection. + * + * The actions done in this subclass used to be done by copying + * org.apache.commons.HttpMethodBase, overlaying our version in place of the + * one that came w/ httpclient. Here is the patch of the difference between + * shipped httpclient code and our mods: + * 

+ *    -- -1338,6 +1346,12 --
+ *
+ *        public void releaseConnection() {
+ *
+ *   +        // HERITRIX always ants the streams closed.
+ *   +        if (responseConnection != null)
+ *   +        {
+ *   +            responseConnection.close();
+ *   +        }
+ *   +
+ *            if (responseStream != null) {
+ *                try {
+ *                    // FYI - this may indirectly invoke responseBodyConsumed.
+ *   -- -1959,6 +1973,11 --
+ *                        this.statusLine = null;
+ *                    }
+ *                }
+ *   +            // HERITRIX mark transition from header to content.
+ *   +            if (this.httpRecorder != null)
+ *   +            {
+ *   +                this.httpRecorder.markContentBegin();
+ *   +            }
+ *                readResponseBody(state, conn);
+ *                processResponseBody(state, conn);
+ *            } catch (IOException e) {
+ *
+ * + *

We're not supposed to have access to the underlying connection object; + * am only violating contract because see cases where httpclient is skipping + * out w/o cleaning up after itself. + * + * @author stack + * @version $Revision$, $Date$ + */ +public class HttpRecorderGetMethod extends GetMethod { + + protected static Logger logger = + Logger.getLogger(HttpRecorderGetMethod.class.getName()); + + /** + * Instance of http recorder method. + */ + protected HttpRecorderMethod httpRecorderMethod = null; + + + public HttpRecorderGetMethod(String uri, Recorder recorder) { + super(uri); + this.httpRecorderMethod = new HttpRecorderMethod(recorder); + } + + protected void readResponseBody(HttpState state, HttpConnection connection) + throws IOException, HttpException { + // We're about to read the body. Mark transition in http recorder. + this.httpRecorderMethod.markContentBegin(connection); + super.readResponseBody(state, connection); + } + + protected boolean shouldCloseConnection(HttpConnection conn) { + // Always close connection after each request. As best I can tell, this + // is superfluous -- we've set our client to be HTTP/1.0. Doing this + // out of paranoia. + return true; + } + + public int execute(HttpState state, HttpConnection conn) + throws HttpException, IOException { + // Save off the connection so we can close it on our way out in case + // httpclient fails to (We're not supposed to have access to the + // underlying connection object; am only violating contract because + // see cases where httpclient is skipping out w/o cleaning up + // after itself). + this.httpRecorderMethod.setConnection(conn); + return super.execute(state, conn); + } + + protected void addProxyConnectionHeader(HttpState state, HttpConnection conn) + throws IOException, HttpException { + super.addProxyConnectionHeader(state, conn); + this.httpRecorderMethod.handleAddProxyConnectionHeader(this); + } +} diff --git a/commons/src/main/java/org/archive/httpclient/HttpRecorderMethod.java b/commons/src/main/java/org/archive/httpclient/HttpRecorderMethod.java new file mode 100644 index 00000000..29dcfcdc --- /dev/null +++ b/commons/src/main/java/org/archive/httpclient/HttpRecorderMethod.java @@ -0,0 +1,111 @@ +/* HttpRecorderMethod + * + * Created on August 22, 2004 + * + * Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.httpclient; + +import java.util.logging.Logger; + +import org.apache.commons.httpclient.Header; +import org.apache.commons.httpclient.HttpConnection; +import org.apache.commons.httpclient.HttpMethod; +import org.archive.util.Recorder; + + +/** + * This class encapsulates the specializations supplied by the + * overrides {@link HttpRecorderGetMethod} and {@link HttpRecorderPostMethod}. + * + * It keeps instance of HttpRecorder and HttpConnection. + * + * @author stack + * @version $Revision$, $Date$ + */ +public class HttpRecorderMethod { + protected static Logger logger = + Logger.getLogger(HttpRecorderMethod.class.getName()); + + /** + * Instance of http recorder we're using recording this http get. + */ + private Recorder httpRecorder = null; + + /** + * Save around so can force close. + * + * See [ 922080 ] IllegalArgumentException (size is wrong). + * https://sourceforge.net/tracker/?func=detail&aid=922080&group_id=73833&atid=539099 + */ + private HttpConnection connection = null; + + + public HttpRecorderMethod(Recorder recorder) { + this.httpRecorder = recorder; + } + + public void markContentBegin(HttpConnection c) { + if (c != this.connection) { + // We're checking that we're not being asked to work on + // a connection that is other than the one we started + // this method#execute with. + throw new IllegalArgumentException("Connections differ: " + + this.connection + " " + c + " " + + Thread.currentThread().getName()); + } + this.httpRecorder.markContentBegin(); + } + + /** + * @return Returns the connection. + */ + public HttpConnection getConnection() { + return this.connection; + } + + /** + * @param connection The connection to set. + */ + public void setConnection(HttpConnection connection) { + this.connection = connection; + } + /** + * @return Returns the httpRecorder. + */ + public Recorder getHttpRecorder() { + return httpRecorder; + } + + /** + * If a 'Proxy-Connection' header has been added to the request, + * it'll be of a 'keep-alive' type. Until we support 'keep-alives', + * override the Proxy-Connection setting and instead pass a 'close' + * (Otherwise every request has to timeout before we notice + * end-of-document). + * @param method Method to find proxy-connection header in. + */ + public void handleAddProxyConnectionHeader(HttpMethod method) { + Header h = method.getRequestHeader("Proxy-Connection"); + if (h != null) { + h.setValue("close"); + method.setRequestHeader(h); + } + } +} diff --git a/commons/src/main/java/org/archive/httpclient/HttpRecorderPostMethod.java b/commons/src/main/java/org/archive/httpclient/HttpRecorderPostMethod.java new file mode 100644 index 00000000..8178ad2b --- /dev/null +++ b/commons/src/main/java/org/archive/httpclient/HttpRecorderPostMethod.java @@ -0,0 +1,86 @@ +/* HttpRecorderPostMethod + * + * Created on Feb 24, 2004 + * + * Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.httpclient; + +import java.io.IOException; + +import org.apache.commons.httpclient.HttpConnection; +import org.apache.commons.httpclient.HttpException; +import org.apache.commons.httpclient.HttpState; +import org.apache.commons.httpclient.methods.PostMethod; +import org.archive.util.Recorder; + + +/** + * Override of PostMethod that marks the passed HttpRecorder w/ the transition + * from HTTP head to body and that forces a close on the responseConnection. + * + * This is a copy of {@link HttpRecorderGetMethod}. Only difference is the + * parent subclass. + * + * @author stack + * @version $Date$ $Revision$ + */ +public class HttpRecorderPostMethod extends PostMethod { + /** + * Instance of http recorder method. + */ + protected HttpRecorderMethod httpRecorderMethod = null; + + + public HttpRecorderPostMethod(String uri, Recorder recorder) { + super(uri); + this.httpRecorderMethod = new HttpRecorderMethod(recorder); + } + + protected void readResponseBody(HttpState state, HttpConnection connection) + throws IOException, HttpException { + // We're about to read the body. Mark transition in http recorder. + this.httpRecorderMethod.markContentBegin(connection); + super.readResponseBody(state, connection); + } + + protected boolean shouldCloseConnection(HttpConnection conn) { + // Always close connection after each request. As best I can tell, this + // is superfluous -- we've set our client to be HTTP/1.0. Doing this + // out of paranoia. + return true; + } + + public int execute(HttpState state, HttpConnection conn) + throws HttpException, IOException { + // Save off the connection so we can close it on our way out in case + // httpclient fails to (We're not supposed to have access to the + // underlying connection object; am only violating contract because + // see cases where httpclient is skipping out w/o cleaning up + // after itself). + this.httpRecorderMethod.setConnection(conn); + return super.execute(state, conn); + } + + protected void addProxyConnectionHeader(HttpState state, HttpConnection conn) + throws IOException, HttpException { + super.addProxyConnectionHeader(state, conn); + this.httpRecorderMethod.handleAddProxyConnectionHeader(this); + } +} diff --git a/commons/src/main/java/org/archive/httpclient/SingleHttpConnectionManager.java b/commons/src/main/java/org/archive/httpclient/SingleHttpConnectionManager.java new file mode 100644 index 00000000..1132b1b8 --- /dev/null +++ b/commons/src/main/java/org/archive/httpclient/SingleHttpConnectionManager.java @@ -0,0 +1,76 @@ +/* SingleHttpConnectionManager +* +* $Id$ +* +* Created on Mar 8, 2004 +* +* Copyright (C) 2004 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.httpclient; + +import java.io.IOException; +import java.io.InputStream; + +import org.apache.commons.httpclient.HostConfiguration; +import org.apache.commons.httpclient.HttpConnection; +import org.apache.commons.httpclient.SimpleHttpConnectionManager; + +/** + * An HttpClient-compatible HttpConnection "manager" that actually + * just gives out a new connection each time -- skipping the overhead + * of connection management, since we already throttle our crawler + * with external mechanisms. + * + * @author gojomo + */ +public class SingleHttpConnectionManager extends SimpleHttpConnectionManager { + + public SingleHttpConnectionManager() { + super(); + } + + public HttpConnection getConnectionWithTimeout( + HostConfiguration hostConfiguration, long timeout) { + + HttpConnection conn = new HttpConnection(hostConfiguration); + conn.setHttpConnectionManager(this); + conn.getParams().setDefaults(this.getParams()); + return conn; + } + + public void releaseConnection(HttpConnection conn) { + // ensure connection is closed + conn.close(); + finishLast(conn); + } + + static void finishLast(HttpConnection conn) { + // copied from superclass because it wasn't made available to subclasses + InputStream lastResponse = conn.getLastResponseInputStream(); + if (lastResponse != null) { + conn.setLastResponseInputStream(null); + try { + lastResponse.close(); + } catch (IOException ioe) { + //FIXME: badness - close to force reconnect. + conn.close(); + } + } + } +} diff --git a/commons/src/main/java/org/archive/httpclient/ThreadLocalHttpConnectionManager.java b/commons/src/main/java/org/archive/httpclient/ThreadLocalHttpConnectionManager.java new file mode 100644 index 00000000..d26cbc40 --- /dev/null +++ b/commons/src/main/java/org/archive/httpclient/ThreadLocalHttpConnectionManager.java @@ -0,0 +1,291 @@ +/** + * ==================================================================== + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + * + */ +package org.archive.httpclient; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.httpclient.HostConfiguration; +import org.apache.commons.httpclient.HttpConnection; +import org.apache.commons.httpclient.HttpConnectionManager; +import org.apache.commons.httpclient.params.HttpConnectionManagerParams; + +/** + * A simple, but thread-safe HttpClient {@link HttpConnectionManager}. + * Based on {@link org.apache.commons.httpclient.SimpleHttpConnectionManager}. + * + * Java >= 1.4 is recommended. + * + * @author Christian Kohlschuetter + */ +public final class ThreadLocalHttpConnectionManager implements + HttpConnectionManager { + + private static final CloserThread closer = new CloserThread(); + private static final Logger logger = Logger + .getLogger(ThreadLocalHttpConnectionManager.class.getName()); + + private final ThreadLocal tl = new ThreadLocal() { + protected synchronized ConnectionInfo initialValue() { + return new ConnectionInfo(); + } + }; + + private ConnectionInfo getConnectionInfo() { + return (ConnectionInfo) tl.get(); + } + + private final class ConnectionInfo { + /** The http connection */ + private HttpConnection conn = null; + + /** + * The time the connection was made idle. + */ + private long idleStartTime = Long.MAX_VALUE; + } + + public ThreadLocalHttpConnectionManager() { + } + + /** + * Since the same connection is about to be reused, make sure the + * previous request was completely processed, and if not + * consume it now. + * @param conn The connection + * @return true, if the connection is reusable + */ + private static boolean finishLastResponse(final HttpConnection conn) { + InputStream lastResponse = conn.getLastResponseInputStream(); + if(lastResponse != null) { + conn.setLastResponseInputStream(null); + try { + lastResponse.close(); + return true; + } catch (IOException ioe) { + // force reconnect. + return false; + } + } else { + return false; + } + } + + /** + * Collection of parameters associated with this connection manager. + */ + private HttpConnectionManagerParams params = new HttpConnectionManagerParams(); + + /** + * @see HttpConnectionManager#getConnection(HostConfiguration) + */ + public HttpConnection getConnection( + final HostConfiguration hostConfiguration) { + return getConnection(hostConfiguration, 0); + } + + /** + * Gets the staleCheckingEnabled value to be set on HttpConnections that are created. + * + * @return true if stale checking will be enabled on HttpConections + * + * @see HttpConnection#isStaleCheckingEnabled() + * + * @deprecated Use {@link HttpConnectionManagerParams#isStaleCheckingEnabled()}, + * {@link HttpConnectionManager#getParams()}. + */ + public boolean isConnectionStaleCheckingEnabled() { + return this.params.isStaleCheckingEnabled(); + } + + /** + * Sets the staleCheckingEnabled value to be set on HttpConnections that are created. + * + * @param connectionStaleCheckingEnabled true if stale checking will be enabled + * on HttpConections + * + * @see HttpConnection#setStaleCheckingEnabled(boolean) + * + * @deprecated Use {@link HttpConnectionManagerParams#setStaleCheckingEnabled(boolean)}, + * {@link HttpConnectionManager#getParams()}. + */ + public void setConnectionStaleCheckingEnabled( + final boolean connectionStaleCheckingEnabled) { + this.params.setStaleCheckingEnabled(connectionStaleCheckingEnabled); + } + + /** + * @see HttpConnectionManager#getConnectionWithTimeout(HostConfiguration, long) + * + * @since 3.0 + */ + public HttpConnection getConnectionWithTimeout( + final HostConfiguration hostConfiguration, final long timeout) { + + final ConnectionInfo ci = getConnectionInfo(); + HttpConnection httpConnection = ci.conn; + + // make sure the host and proxy are correct for this connection + // close it and set the values if they are not + if(httpConnection == null || !finishLastResponse(httpConnection) + || !hostConfiguration.hostEquals(httpConnection) + || !hostConfiguration.proxyEquals(httpConnection)) { + + if(httpConnection != null && httpConnection.isOpen()) { + closer.closeConnection(httpConnection); + } + + httpConnection = new HttpConnection(hostConfiguration); + httpConnection.setHttpConnectionManager(this); + httpConnection.getParams().setDefaults(this.params); + ci.conn = httpConnection; + + httpConnection.setHost(hostConfiguration.getHost()); + httpConnection.setPort(hostConfiguration.getPort()); + httpConnection.setProtocol(hostConfiguration.getProtocol()); + httpConnection.setLocalAddress(hostConfiguration.getLocalAddress()); + + httpConnection.setProxyHost(hostConfiguration.getProxyHost()); + httpConnection.setProxyPort(hostConfiguration.getProxyPort()); + } + + // remove the connection from the timeout handler + ci.idleStartTime = Long.MAX_VALUE; + + return httpConnection; + } + + /** + * @see HttpConnectionManager#getConnection(HostConfiguration, long) + * + * @deprecated Use #getConnectionWithTimeout(HostConfiguration, long) + */ + public HttpConnection getConnection( + final HostConfiguration hostConfiguration, final long timeout) { + return getConnectionWithTimeout(hostConfiguration, timeout); + } + + /** + * @see HttpConnectionManager#releaseConnection(org.apache.commons.httpclient.HttpConnection) + */ + public void releaseConnection(final HttpConnection conn) { + final ConnectionInfo ci = getConnectionInfo(); + HttpConnection httpConnection = ci.conn; + + if(conn != httpConnection) { + throw new IllegalStateException( + "Unexpected release of an unknown connection."); + } + + finishLastResponse(httpConnection); + + // track the time the connection was made idle + ci.idleStartTime = System.currentTimeMillis(); + } + + /** + * Returns {@link HttpConnectionManagerParams parameters} associated + * with this connection manager. + * + * @since 2.1 + * + * @see HttpConnectionManagerParams + */ + public HttpConnectionManagerParams getParams() { + return this.params; + } + + /** + * Assigns {@link HttpConnectionManagerParams parameters} for this + * connection manager. + * + * @since 2.1 + * + * @see HttpConnectionManagerParams + */ + public void setParams(final HttpConnectionManagerParams p) { + if(p == null) { + throw new IllegalArgumentException("Parameters may not be null"); + } + this.params = p; + } + + /** + * @since 3.0 + */ + public void closeIdleConnections(final long idleTimeout) { + long maxIdleTime = System.currentTimeMillis() - idleTimeout; + + final ConnectionInfo ci = getConnectionInfo(); + + if(ci.idleStartTime <= maxIdleTime) { + ci.conn.close(); + } + } + + private static final class CloserThread extends Thread { + private List connections + = new ArrayList(); + + private static final int SLEEP_INTERVAL = 5000; + + public CloserThread() { + super("HttpConnection closer"); + // Make this a daemon thread so it can't be responsible for the JVM + // not shutting down. + setDaemon(true); + start(); + } + + public void closeConnection(final HttpConnection conn) { + synchronized (connections) { + connections.add(conn); + } + } + + public void run() { + try { + while (!Thread.interrupted()) { + Thread.sleep(SLEEP_INTERVAL); + + List s; + synchronized (connections) { + s = connections; + connections = new ArrayList(); + } + logger.log(Level.INFO, "Closing " + s.size() + + " HttpConnections"); + for(final Iterator it = s.iterator(); + it.hasNext();) { + HttpConnection conn = it.next(); + conn.close(); + conn.setHttpConnectionManager(null); + it.remove(); + } + } + } catch (InterruptedException e) { + return; + } + } + } +} diff --git a/commons/src/main/java/org/archive/httpclient/package.html b/commons/src/main/java/org/archive/httpclient/package.html new file mode 100644 index 00000000..87ae77ed --- /dev/null +++ b/commons/src/main/java/org/archive/httpclient/package.html @@ -0,0 +1,24 @@ + + + +org.archive.httpclient package + +Provides specializations on + apache jakarta + commons httpclient. + +

HttpRecorderGetMethod

+

Class that the passed HttpRecorder w/ boundary between + HTTP header and content. Also forces a close on the response on + call to releaseConnection.

+ +

ConfigurableTrustManagerProtocolSocketFactory

+

A protocol socket factory that allows setting of trust level on + construction.

+ +

References

+

JavaTM Secure Socket Extension (JSSE): Reference Guide

+ + + diff --git a/commons/src/main/java/org/archive/io/Arc2Warc.java b/commons/src/main/java/org/archive/io/Arc2Warc.java new file mode 100644 index 00000000..552e5bde --- /dev/null +++ b/commons/src/main/java/org/archive/io/Arc2Warc.java @@ -0,0 +1,208 @@ +/* $Id$ + * + * Created Aug 29, 2006 + * + * Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.io; + +import java.io.BufferedOutputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; +import org.archive.io.arc.ARCConstants; +import org.archive.io.arc.ARCReader; +import org.archive.io.arc.ARCReaderFactory; +import org.archive.io.arc.ARCRecord; +import org.archive.io.warc.WARCConstants; +import org.archive.io.warc.WARCWriter; +import org.archive.util.FileUtils; +import org.archive.util.anvl.ANVLRecord; + + +/** + * Convert ARCs to (sortof) WARCs. + * @author stack + * @version $Date$ $Revision$ + */ +public class Arc2Warc { + private static void usage(HelpFormatter formatter, Options options, + int exitCode) { + formatter.printHelp("java org.archive.io.arc.Arc2Warc " + + "[--force] ARC_INPUT WARC_OUTPUT", options); + System.exit(exitCode); + } + + private static String getRevision() { + return Warc2Arc.parseRevision("$Revision$"); + } + + public void transform(final File arc, final File warc, final boolean force) + throws IOException { + FileUtils.assertReadable(arc); + if (warc.exists() && !force) { + throw new IOException("Target WARC already exists. " + + "Will not overwrite."); + } + + ARCReader reader = ARCReaderFactory.get(arc, false, 0); + transform(reader, warc); + } + + protected void transform(final ARCReader reader, final File warc) + throws IOException { + WARCWriter writer = null; + // No point digesting. Digest is available after reading of ARC which + // is too late for inclusion in WARC. + reader.setDigest(false); + try { + BufferedOutputStream bos = + new BufferedOutputStream(new FileOutputStream(warc)); + // Get the body of the first ARC record as a String so can dump it + // into first record of WARC. + final Iterator i = reader.iterator(); + ARCRecord firstRecord = (ARCRecord)i.next(); + ByteArrayOutputStream baos = + new ByteArrayOutputStream((int)firstRecord.getHeader(). + getLength()); + firstRecord.dump(baos); + // Add ARC first record content as an ANVLRecord. + ANVLRecord ar = new ANVLRecord(1); + ar.addLabelValue("Filedesc", baos.toString()); + List metadata = new ArrayList(1); + metadata.add(ar.toString()); + // Now create the writer. If reader was compressed, lets write + // a compressed WARC. + writer = new WARCWriter(null, bos, warc, + reader.isCompressed(), null, metadata); + // Write a warcinfo record with description about how this WARC + // was made. + writer.writeWarcinfoRecord(warc.getName(), + "Made from " + reader.getReaderIdentifier() + " by " + + this.getClass().getName() + "/" + getRevision()); + for (; i.hasNext();) { + write(writer, (ARCRecord)i.next()); + } + } finally { + if (reader != null) { + reader.close(); + } + if (writer != null) { + // I don't want the close being logged -- least, not w/o log of + // an opening (and that'd be a little silly for simple script + // like this). Currently, it logs at level INFO so that close + // of files gets written to log files. Up the log level just + // for the close. + Logger l = Logger.getLogger(writer.getClass().getName()); + Level oldLevel = l.getLevel(); + l.setLevel(Level.WARNING); + try { + writer.close(); + } finally { + l.setLevel(oldLevel); + } + } + } + } + + protected void write(final WARCWriter writer, + final ARCRecord r) + throws IOException { + ANVLRecord ar = new ANVLRecord(); + String ip = (String)r.getHeader(). + getHeaderValue((ARCConstants.IP_HEADER_FIELD_KEY)); + if (ip != null && ip.length() > 0) { + ar.addLabelValue(WARCConstants.NAMED_FIELD_IP_LABEL, ip); + } + // If contentBody > 0, assume http headers. Make the mimetype + // be application/http. Otherwise, give it ARC mimetype. + writer.writeResourceRecord(r.getHeader().getUrl(), + r.getHeader().getDate(), + (r.getHeader().getContentBegin() > 0)? + WARCConstants.HTTP_RESPONSE_MIMETYPE: + r.getHeader().getMimetype(), + ar, r, r.getHeader().getLength()); + } + + /** + * Command-line interface to Arc2Warc. + * + * @param args Command-line arguments. + * @throws ParseException Failed parse of the command line. + * @throws IOException + * @throws java.text.ParseException + */ + @SuppressWarnings("unchecked") +public static void main(String [] args) + throws ParseException, IOException, java.text.ParseException { + Options options = new Options(); + options.addOption(new Option("h","help", false, + "Prints this message and exits.")); + options.addOption(new Option("f","force", false, + "Force overwrite of target file.")); + PosixParser parser = new PosixParser(); + CommandLine cmdline = parser.parse(options, args, false); + List cmdlineArgs = cmdline.getArgList(); + Option [] cmdlineOptions = cmdline.getOptions(); + HelpFormatter formatter = new HelpFormatter(); + + // If no args, print help. + if (cmdlineArgs.size() <= 0) { + usage(formatter, options, 0); + } + + // Now look at options passed. + boolean force = false; + for (int i = 0; i < cmdlineOptions.length; i++) { + switch(cmdlineOptions[i].getId()) { + case 'h': + usage(formatter, options, 0); + break; + + case 'f': + force = true; + break; + + default: + throw new RuntimeException("Unexpected option: " + + + cmdlineOptions[i].getId()); + } + } + + // If no args, print help. + if (cmdlineArgs.size() != 2) { + usage(formatter, options, 0); + } + (new Arc2Warc()).transform(new File(cmdlineArgs.get(0).toString()), + new File(cmdlineArgs.get(1).toString()), force); + } +} diff --git a/commons/src/main/java/org/archive/io/ArchiveFileConstants.java b/commons/src/main/java/org/archive/io/ArchiveFileConstants.java new file mode 100644 index 00000000..1d5d973a --- /dev/null +++ b/commons/src/main/java/org/archive/io/ArchiveFileConstants.java @@ -0,0 +1,118 @@ +/* $Id$ + * + * Created on August 16th, 2006. + * + * Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.io; + +/** + * Constants used by Archive files and in Archive file processing. + * @author stack + * @version $Date$ $Revision$ + */ +public interface ArchiveFileConstants { + /** + * Suffix given to files currently in use. + */ + public static final String OCCUPIED_SUFFIX = ".open"; + + /** + * Suffix appended to 'broken' files. + */ + public static final String INVALID_SUFFIX = ".invalid"; + + /** + * Compressed file extention. + */ + public static final String COMPRESSED_FILE_EXTENSION = "gz"; + + /** + * Dot plus compressed file extention. + */ + public static final String DOT_COMPRESSED_FILE_EXTENSION = "." + + COMPRESSED_FILE_EXTENSION; + + /** + * Key for the Archive File version field. + */ + public static final String VERSION_FIELD_KEY = "version"; + + /** + * Key for the Archive File length field. + */ + public static final String LENGTH_FIELD_KEY = "length"; + + /** + * Key for the Archive File type field. + */ + public static final String TYPE_FIELD_KEY = "type"; + + /** + * Key for the Archive File URL field. + */ + public static final String URL_FIELD_KEY = "subject-uri"; + + /** + * Key for the Archive File Creation Date field. + */ + public static final String DATE_FIELD_KEY = "creation-date"; + + /** + * Key for the Archive File mimetype field. + */ + public static final String MIMETYPE_FIELD_KEY = "content-type"; + + /** + * Key for the Archive File record field. + */ + public static final String RECORD_IDENTIFIER_FIELD_KEY = + "record-identifier"; + + /** + * Key for the Archive Record absolute offset into Archive file. + */ + public static final String ABSOLUTE_OFFSET_KEY = "absolute-offset"; + + public static final String READER_IDENTIFIER_FIELD_KEY = + "reader-identifier"; + + /** + * Size used to preallocate stringbuffer used outputting a cdx line. + * The numbers below are guesses at sizes of each of the cdx fields. + * The ones in the below are spaces. Here is the legend used outputting + * the cdx line: CDX b e a m s c V n g. Consult cdx documentation on + * meaning of each of these fields. + */ + public static final int CDX_LINE_BUFFER_SIZE = 14 + 1 + 15 + 1 + 1024 + + 1 + 24 + 1 + + 3 + 1 + 32 + 1 + 20 + 1 + 20 + 1 + 64; + + public static final String DEFAULT_DIGEST_METHOD = "SHA-1"; + + public static final char SINGLE_SPACE = ' '; + + public static final String CRLF = "\r\n"; + + public static final String CDX = "cdx"; + public static final String DUMP = "dump"; + public static final String GZIP_DUMP = "gzipdump"; + public static final String HEADER = "header"; + public static final String NOHEAD = "nohead"; + public static final String CDX_FILE = "cdxfile"; +} diff --git a/commons/src/main/java/org/archive/io/ArchiveReader.java b/commons/src/main/java/org/archive/io/ArchiveReader.java new file mode 100644 index 00000000..259987ff --- /dev/null +++ b/commons/src/main/java/org/archive/io/ArchiveReader.java @@ -0,0 +1,779 @@ +/* $Id$ + * + * Created on August 21st, 2006 + * + * Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.io; + +import it.unimi.dsi.fastutil.io.RepositionableStream; + +import java.io.BufferedInputStream; +import java.io.BufferedWriter; +import java.io.EOFException; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.archive.util.MimetypeUtils; + + +/** + * Reader for an Archive file of Archive {@link ArchiveRecord}s. + * @author stack + * @version $Date$ $Version$ + */ +public abstract class ArchiveReader implements ArchiveFileConstants, Iterable { + /** + * Is this Archive file compressed? + */ + private boolean compressed = false; + + /** + * Should we digest as we read? + */ + private boolean digest = true; + + /** + * Should the parse be strict? + */ + private boolean strict = false; + + /** + * Archive file input stream. + * + * Keep it around so we can close it when done. + * + *

Set in constructor. Must support {@link RepositionableStream} + * interface. Make it protected so subclasses have access. + */ + private InputStream in = null; + + /** + * Maximum amount of recoverable exceptions in a row. + * If more than this amount in a row, we'll let out the exception rather + * than go back in for yet another retry. + */ + public static final int MAX_ALLOWED_RECOVERABLES = 10; + + + /** + * The Record currently being read. + * + * Keep this ongoing reference so we'll close the record even if the caller + * doesn't. + */ + private ArchiveRecord currentRecord = null; + + /** + * Descriptive string for the Archive file we're going against: + * full path, url, etc. -- depends on context in which file was made. + */ + private String identifier = null; + + /** + * Archive file version. + */ + private String version = null; + + + protected ArchiveReader() { + super(); + } + + /** + * Convenience method used by subclass constructors. + * @param i Identifier for Archive file this reader goes against. + */ + protected void initialize(final String i) { + setReaderIdentifier(i); + } + + /** + * Convenience method for constructors. + * + * @param f File to read. + * @param offset Offset at which to start reading. + * @return InputStream to read from. + * @throws IOException If failed open or fail to get a memory + * mapped byte buffer on file. + */ + protected InputStream getInputStream(final File f, final long offset) + throws IOException { + return new RandomAccessBufferedInputStream( + new RandomAccessInputStream(f, offset)); + } + + public boolean isCompressed() { + return this.compressed; + } + + /** + * Get record at passed offset. + * + * @param offset Byte index into file at which a record starts. + * @return An Archive Record reference. + * @throws IOException + */ + public ArchiveRecord get(long offset) throws IOException { + cleanupCurrentRecord(); + RepositionableStream ps = (RepositionableStream)this.in; + long currentOffset = ps.position(); + if (currentOffset != offset) { + currentOffset = offset; + ps.position(offset); + } + return createArchiveRecord(this.in, currentOffset); + } + + /** + * @return Return Archive Record created against current offset. + * @throws IOException + */ + public ArchiveRecord get() throws IOException { + return createArchiveRecord(this.in, + ((RepositionableStream)this.in).position()); + } + + public void close() throws IOException { + if (this.in != null) { + this.in.close(); + this.in = null; + } + } + + /** + * Rewinds stream to start of the Archive file. + * @throws IOException if stream is not resettable. + */ + protected void rewind() throws IOException { + cleanupCurrentRecord(); + if (this.in instanceof RepositionableStream) { + try { + ((RepositionableStream)this.in).position(0); + } catch (IOException e) { + throw new RuntimeException(e); + } + } else { + throw new IOException("Stream is not resettable."); + } + } + + /** + * Cleanout the current record if there is one. + * @throws IOException + */ + protected void cleanupCurrentRecord() throws IOException { + if (this.currentRecord != null) { + this.currentRecord.close(); + gotoEOR(this.currentRecord); + this.currentRecord = null; + } + } + + /** + * Return an Archive Record homed on offset into + * is. + * @param is Stream to read Record from. + * @param offset Offset to find Record at. + * @return ArchiveRecord instance. + * @throws IOException + */ + protected abstract ArchiveRecord createArchiveRecord(InputStream is, + long offset) + throws IOException; + + /** + * Skip over any trailing new lines at end of the record so we're lined up + * ready to read the next. + * @param record + * @throws IOException + */ + protected abstract void gotoEOR(ArchiveRecord record) throws IOException; + + public abstract String getFileExtension(); + public abstract String getDotFileExtension(); + + /** + * @return Version of this Archive file. + */ + public String getVersion() { + return this.version; + } + + /** + * Validate the Archive file. + * + * This method iterates over the file throwing exception if it fails + * to successfully parse any record. + * + *

Assumes the stream is at the start of the file. + * @return List of all read Archive Headers. + * + * @throws IOException + */ + public List validate() throws IOException { + return validate(-1); + } + + /** + * Validate the Archive file. + * + * This method iterates over the file throwing exception if it fails + * to successfully parse. + * + *

We start validation from whereever we are in the stream. + * + * @param noRecords Number of records expected. Pass -1 if number is + * unknown. + * + * @return List of all read metadatas. As we validate records, we add + * a reference to the read metadata. + * + * @throws IOException + */ + public List validate(int noRecords) + throws IOException { + List hs = new ArrayList(); + int count = 0; + setStrict(true); + for (Iterator i = iterator(); i.hasNext();) { + count++; + ArchiveRecord r = i.next(); + if (r.getHeader().getLength() <= 0 + && r.getHeader().getMimetype(). + equals(MimetypeUtils.NO_TYPE_MIMETYPE)) { + throw new IOException("ARCRecord content is empty."); + } + r.close(); + // Add reference to metadata into a list of metadatas. + hs.add(r.getHeader()); + } + + if (noRecords != -1) { + if (count != noRecords) { + throw new IOException("Count of records, " + + Integer.toString(count) + " is less than expected " + + Integer.toString(noRecords)); + } + } + + return hs; + } + + /** + * Test Archive file is valid. + * Assumes the stream is at the start of the file. Be aware that this + * method makes a pass over the whole file. + * @return True if file can be successfully parsed. + */ + public boolean isValid() { + boolean valid = false; + try { + validate(); + valid = true; + } catch(Exception e) { + // File is not valid if exception thrown parsing. + valid = false; + } + + return valid; + } + + /** + * @return Returns the strict. + */ + public boolean isStrict() { + return this.strict; + } + + /** + * @param s The strict to set. + */ + public void setStrict(boolean s) { + this.strict = s; + } + + /** + * @param d True if we're to digest. + */ + public void setDigest(boolean d) { + this.digest = d; + } + + /** + * @return True if we're digesting as we read. + */ + public boolean isDigest() { + return this.digest; + } + + protected Logger getLogger() { + return Logger.getLogger(this.getClass().getName()); + } + + protected InputStream getInputStream() { + return this.in; + } + + /** + * Returns an ArchiveRecord iterator. + * Of note, on IOException, especially if ZipException reading compressed + * ARCs, rather than fail the iteration, try moving to the next record. + * If {@link ArchiveReader#strict} is not set, this will usually succeed. + * @return An iterator over ARC records. + */ + public Iterator iterator() { + // Eat up any record outstanding. + try { + cleanupCurrentRecord(); + } catch (IOException e) { + throw new RuntimeException(e); + } + + // Now reset stream to the start of the arc file. + try { + rewind(); + } catch (IOException e) { + throw new RuntimeException(e); + } + return new ArchiveRecordIterator(); + } + + protected void setCompressed(boolean compressed) { + this.compressed = compressed; + } + + /** + * @return The current ARC record or null if none. + * After construction has the arcfile header record. + * @see #get() + */ + protected ArchiveRecord getCurrentRecord() { + return this.currentRecord; + } + + protected ArchiveRecord currentRecord(final ArchiveRecord currentRecord) { + this.currentRecord = currentRecord; + return currentRecord; + } + + protected InputStream getIn() { + return in; + } + + protected void setIn(InputStream in) { + this.in = in; + } + + protected void setVersion(String version) { + this.version = version; + } + + public String getReaderIdentifier() { + return this.identifier; + } + + protected void setReaderIdentifier(final String i) { + this.identifier = i; + } + + /** + * Log on stderr. + * Logging should go via the logging system. This method + * bypasses the logging system going direct to stderr. + * Should not generally be used. Its used for rare messages + * that come of cmdline usage of ARCReader ERRORs and WARNINGs. + * Override if using ARCReader in a context where no stderr or + * where you'd like to redirect stderr to other than System.err. + * @param level Level to log message at. + * @param message Message to log. + */ + public void logStdErr(Level level, String message) { + System.err.println(level.toString() + " " + message); + } + + /** + * Add buffering to RandomAccessInputStream. + */ + protected class RandomAccessBufferedInputStream + extends BufferedInputStream implements RepositionableStream { + + public RandomAccessBufferedInputStream(RandomAccessInputStream is) + throws IOException { + super(is); + } + + public RandomAccessBufferedInputStream(RandomAccessInputStream is, int size) + throws IOException { + super(is, size); + } + + public long position() throws IOException { + // Current position is the underlying files position + // minus the amount thats in the buffer yet to be read. + return ((RandomAccessInputStream)this.in).position() - + (this.count - this.pos); + } + + public void position(long position) throws IOException { + // Force refill of buffer whenever there's been a seek. + this.pos = 0; + this.count = 0; + ((RandomAccessInputStream)this.in).position(position); + } + + public int available() throws IOException { + // Avoid overflow on large datastreams + long amount = (long)in.available() + (long)(count - pos); + return (amount >= Integer.MAX_VALUE)? Integer.MAX_VALUE: (int)amount; + } + } + + /** + * Inner ArchiveRecord Iterator class. + * Throws RuntimeExceptions in {@link #hasNext()} and {@link #next()} if + * trouble pulling record from underlying stream. + * @author stack + */ + protected class ArchiveRecordIterator implements Iterator { + private final Logger logger = + Logger.getLogger(this.getClass().getName()); + /** + * @return True if we have more records to read. + * @exception RuntimeException Can throw an IOException wrapped in a + * RuntimeException if a problem reading underlying stream (Corrupted + * gzip, etc.). + */ + public boolean hasNext() { + // Call close on any extant record. This will scoot us past + // any content not yet read. + try { + cleanupCurrentRecord(); + } catch (IOException e) { + if (isStrict()) { + throw new RuntimeException(e); + } + if (e instanceof EOFException) { + logger.warning("Premature EOF cleaning up " + + currentRecord.getHeader().toString() + ": " + + e.getMessage()); + return false; + } + // If not strict, try going again. We might be able to skip + // over the bad record. + logger.warning("Trying skip of failed record cleanup of " + + currentRecord.getHeader().toString() + ": " + + e.getMessage()); + } + return innerHasNext(); + } + + protected boolean innerHasNext() { + long offset = -1; + try { + offset = ((RepositionableStream)getInputStream()).position(); + return getInputStream().available() > 0; + } catch (IOException e) { + throw new RuntimeException("Offset " + offset, e); + } + } + + /** + * Tries to move to next record if we get + * {@link RecoverableIOException}. If not strict + * tries to move to next record if we get an + * {@link IOException}. + * @return Next object. + * @exception RuntimeException Throws a runtime exception, + * usually a wrapping of an IOException, if trouble getting + * a record (Throws exception rather than return null). + */ + public ArchiveRecord next() { + long offset = -1; + try { + offset = ((RepositionableStream)getInputStream()).position(); + return exceptionNext(); + } catch (IOException e) { + if (!isStrict()) { + // Retry though an IOE. Maybe we will succeed reading + // subsequent record. + try { + if (hasNext()) { + getLogger().warning("Bad Record. Trying skip " + + "(Current offset " + offset + "): " + + e.getMessage()); + return exceptionNext(); + } + // Else we are at last record. Iterator#next is + // expecting value. We do not have one. Throw exception. + throw new RuntimeException("Retried but no next " + + "record (Offset " + offset + ")", e); + } catch (IOException e1) { + throw new RuntimeException("After retry (Offset " + + offset + ")", e1); + } + } + throw new RuntimeException("(Offset " + offset + ")", e); + } + } + + /** + * A next that throws exceptions and has handling of + * recoverable exceptions moving us to next record. Can call + * hasNext which itself may throw exceptions. + * @return Next record. + * @throws IOException + * @throws RuntimeException Thrown when we've reached maximum + * retries. + */ + protected ArchiveRecord exceptionNext() + throws IOException, RuntimeException { + ArchiveRecord result = null; + IOException ioe = null; + for (int i = MAX_ALLOWED_RECOVERABLES; i > 0 && + result == null; i--) { + ioe = null; + try { + result = innerNext(); + } catch (RecoverableIOException e) { + ioe = e; + getLogger().warning(e.getMessage()); + if (hasNext()) { + continue; + } + // No records left. Throw exception rather than + // return null. The caller is expecting to get + // back a record since they've just called + // hasNext. + break; + } + } + if (ioe != null) { + // Then we did MAX_ALLOWED_RECOVERABLES retries. Throw + // the recoverable ioe wrapped in a RuntimeException so + // it goes out pass checks for IOE. + throw new RuntimeException("Retried " + + MAX_ALLOWED_RECOVERABLES + " times in a row", ioe); + } + return result; + } + + protected ArchiveRecord innerNext() throws IOException { + return get(((RepositionableStream)getInputStream()).position()); + } + + public void remove() { + throw new UnsupportedOperationException(); + } + } + + protected static String stripExtension(final String name, + final String ext) { + return (!name.endsWith(ext))? name: + name.substring(0, name.length() - ext.length()); + } + + /** + * @return short name of Archive file. + */ + public String getFileName() { + return (new File(getReaderIdentifier())).getName(); + } + + /** + * @return short name of Archive file. + */ + public String getStrippedFileName() { + return getStrippedFileName(getFileName(), + getDotFileExtension()); + } + + /** + * @param name Name of ARCFile. + * @param dotFileExtension '.arc' or '.warc', etc. + * @return short name of Archive file. + */ + public static String getStrippedFileName(String name, + final String dotFileExtension) { + name = stripExtension(name, + ArchiveFileConstants.DOT_COMPRESSED_FILE_EXTENSION); + return stripExtension(name, dotFileExtension); + } + + /** + * @param value Value to test. + * @return True if value is 'true', else false. + */ + protected static boolean getTrueOrFalse(final String value) { + if (value == null || value.length() <= 0) { + return false; + } + return Boolean.TRUE.toString().equals(value.toLowerCase()); + } + + /** + * @param format Format to use outputting. + * @throws IOException + * @throws java.text.ParseException + * @return True if handled. + */ + protected boolean output(final String format) + throws IOException, java.text.ParseException { + boolean result = true; + // long start = System.currentTimeMillis(); + + // Write output as pseudo-CDX file. See + // http://www.archive.org/web/researcher/cdx_legend.php + // and http://www.archive.org/web/researcher/example_cdx.php. + // Hash is hard-coded straight SHA-1 hash of content. + if (format.equals(DUMP)) { + // No point digesting dumping. + setDigest(false); + dump(false); + } else if (format.equals(GZIP_DUMP)) { + // No point digesting dumping. + setDigest(false); + dump(true); + } else if (format.equals(CDX)) { + cdxOutput(false); + } else if (format.equals(CDX_FILE)) { + cdxOutput(true); + } else { + result = false; + } + return result; + } + + protected void cdxOutput(boolean toFile) + throws IOException { + BufferedWriter cdxWriter = null; + if (toFile) { + String cdxFilename = stripExtension(getReaderIdentifier(), + DOT_COMPRESSED_FILE_EXTENSION); + cdxFilename = stripExtension(cdxFilename, getDotFileExtension()); + cdxFilename += ('.' + CDX); + cdxWriter = new BufferedWriter(new FileWriter(cdxFilename)); + } + + String header = "CDX b e a m s c " + ((isCompressed()) ? "V" : "v") + + " n g"; + if (toFile) { + cdxWriter.write(header); + cdxWriter.newLine(); + } else { + System.out.println(header); + } + + String strippedFileName = getStrippedFileName(); + try { + for (Iterator ii = iterator(); ii.hasNext();) { + ArchiveRecord r = ii.next(); + if (toFile) { + cdxWriter.write(r.outputCdx(strippedFileName)); + cdxWriter.newLine(); + } else { + System.out.println(r.outputCdx(strippedFileName)); + } + } + } finally { + if (toFile) { + cdxWriter.close(); + } + } + } + + /** + * Output passed record using passed format specifier. + * @param format What format to use outputting. + * @throws IOException + * @return True if handled. + */ + public boolean outputRecord(final String format) + throws IOException { + boolean result = true; + if (format.equals(CDX)) { + System.out.println(get().outputCdx(getStrippedFileName())); + } else if(format.equals(ArchiveFileConstants.DUMP)) { + // No point digesting if dumping content. + setDigest(false); + get().dump(); + } else { + result = false; + } + return result; + } + + /** + * Dump this file on STDOUT + * @throws compress True if dumped output is compressed. + * @throws IOException + * @throws java.text.ParseException + */ + public abstract void dump(final boolean compress) + throws IOException, java.text.ParseException; + + /** + * @return an ArchiveReader that will delete a local file on close. Used + * when we bring Archive files local and need to clean up afterward. + */ + public abstract ArchiveReader getDeleteFileOnCloseReader(final File f); + + /** + * Output passed record using passed format specifier. + * @param r ARCReader instance to output. + * @param format What format to use outputting. + * @throws IOException + */ + protected static void outputRecord(final ArchiveReader r, + final String format) + throws IOException { + if (!r.outputRecord(format)) { + throw new IOException("Unsupported format" + + " (or unsupported on a single record): " + format); + } + } + + /** + * @return Base Options object filled out with help, digest, strict, etc. + * options. + */ + protected static Options getOptions() { + Options options = new Options(); + options.addOption(new Option("h","help", false, + "Prints this message and exits.")); + options.addOption(new Option("o","offset", true, + "Outputs record at this offset into file.")); + options.addOption(new Option("d","digest", true, + "Pass true|false. Expensive. Default: true (SHA-1).")); + options.addOption(new Option("s","strict", false, + "Strict mode. Fails parse if incorrectly formatted file.")); + options.addOption(new Option("f","format", true, + "Output options: 'cdx', cdxfile', 'dump', 'gzipdump'," + + "'or 'nohead'. Default: 'cdx'.")); + return options; + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/io/ArchiveReaderFactory.java b/commons/src/main/java/org/archive/io/ArchiveReaderFactory.java new file mode 100644 index 00000000..cf80cc65 --- /dev/null +++ b/commons/src/main/java/org/archive/io/ArchiveReaderFactory.java @@ -0,0 +1,326 @@ +/* $Id$ + * + * Created on August 18th, 2006 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.io; + +import it.unimi.dsi.fastutil.io.RepositionableStream; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLConnection; + +import org.archive.io.arc.ARCReaderFactory; +import org.archive.io.warc.WARCReaderFactory; +import org.archive.net.UURI; +import org.archive.net.md5.Md5URLConnection; +import org.archive.net.rsync.RsyncURLConnection; +import org.archive.util.IoUtils; + + +/** + * Factory that returns an Archive file Reader. + * Returns Readers for ARCs or WARCs. + * @author stack + * @version $Date$ $Revision$ + */ +public class ArchiveReaderFactory implements ArchiveFileConstants { + // Static block to enable S3 URLs + static { + if (System.getProperty("java.protocol.handler.pkgs") != null) { + System.setProperty("java.protocol.handler.pkgs", + System.getProperty("java.protocol.handler.pkgs") + + "|" + "org.archive.net"); + } else { + System.setProperty("java.protocol.handler.pkgs", "org.archive.net"); + } + } + + /** + * Offset value for when we want to stream all. + */ + private final static int STREAM_ALL = -1; + + private static final ArchiveReaderFactory factory = + new ArchiveReaderFactory(); + + /** + * Shutdown any public access to default constructor. + */ + protected ArchiveReaderFactory() { + super(); + } + + /** + * Get an Archive file Reader on passed path or url. + * Does primitive heuristic figuring if path or URL. + * @param arcFileOrUrl File path or URL pointing at an Archive file. + * @return An Archive file Reader. + * @throws IOException + * @throws MalformedURLException + * @throws IOException + */ + public static ArchiveReader get(final String arcFileOrUrl) + throws MalformedURLException, IOException { + return ArchiveReaderFactory.factory.getArchiveReader(arcFileOrUrl); + } + + protected ArchiveReader getArchiveReader(final String arcFileOrUrl) + throws MalformedURLException, IOException { + return getArchiveReader(arcFileOrUrl, STREAM_ALL); + } + + protected ArchiveReader getArchiveReader(final String arcFileOrUrl, + final long offset) + throws MalformedURLException, IOException { + return UURI.hasScheme(arcFileOrUrl)? + get(new URL(arcFileOrUrl), offset): + get(new File(arcFileOrUrl), offset); + } + + /** + * @param f An Archive file to read. + * @return An ArchiveReader + * @throws IOException + */ + public static ArchiveReader get(final File f) throws IOException { + return ArchiveReaderFactory.factory.getArchiveReader(f); + } + + protected ArchiveReader getArchiveReader(final File f) + throws IOException { + return getArchiveReader(f, 0); + } + + /** + * @param f An Archive file to read. + * @param offset Have returned Reader set to start reading at this offset. + * @return An ArchiveReader + * @throws IOException + */ + public static ArchiveReader get(final File f, final long offset) + throws IOException { + return ArchiveReaderFactory.factory.getArchiveReader(f, offset); + } + + protected ArchiveReader getArchiveReader(final File f, + final long offset) + throws IOException { + if (ARCReaderFactory.isARCSuffix(f.getName())) { + return ARCReaderFactory.get(f, true, offset); + } else if (WARCReaderFactory.isWARCSuffix(f.getName())) { + return WARCReaderFactory.get(f, offset); + } + throw new IOException("Unknown file extension (Not ARC nor WARC): " + + f.getName()); + } + + /** + * Wrap a Reader around passed Stream. + * @param s Identifying String for this Stream used in error messages. + * Must be a string that ends with the name of the file we're to put + * an ArchiveReader on. This code looks at file endings to figure + * whether to return an ARC or WARC reader. + * @param is Stream. Stream will be wrapped with implementation of + * RepositionableStream unless already supported. + * @param atFirstRecord Are we at first Record? + * @return ArchiveReader. + * @throws IOException + */ + public static ArchiveReader get(final String s, final InputStream is, + final boolean atFirstRecord) + throws IOException { + return ArchiveReaderFactory.factory.getArchiveReader(s, is, + atFirstRecord); + } + + /** + * @param is + * @return If passed is is + * {@link RepositionableInputStream}, returns is, else we + * wrap is with {@link RepositionableStream}. + */ + protected InputStream asRepositionable(final InputStream is) { + if (is instanceof RepositionableStream) { + return is; + } + // RepositionableInputStream calls mark on each read so can back up at + // least the read amount. Needed for gzip inflater overinflations + // reading into the next gzip member. + return new RepositionableInputStream(is, 16 * 1024); + } + + protected ArchiveReader getArchiveReader(final String id, + final InputStream is, final boolean atFirstRecord) + throws IOException { + final InputStream stream = asRepositionable(is); + if (ARCReaderFactory.isARCSuffix(id)) { + return ARCReaderFactory.get(id, stream, atFirstRecord); + } else if (WARCReaderFactory.isWARCSuffix(id)) { + return WARCReaderFactory.get(id, stream, atFirstRecord); + } + throw new IOException("Unknown extension (Not ARC nor WARC): " + id); + } + + /** + * Get an Archive Reader aligned at offset. + * This version of get will not bring the file local but will try to + * stream across the net making an HTTP 1.1 Range request on remote + * http server (RFC1435 Section 14.35). + * @param u HTTP URL for an Archive file. + * @param offset Offset into file at which to start fetching. + * @return An ArchiveReader aligned at offset. + * @throws IOException + */ + public static ArchiveReader get(final URL u, final long offset) + throws IOException { + return ArchiveReaderFactory.factory.getArchiveReader(u, offset); + } + + protected ArchiveReader getArchiveReader(final URL f, final long offset) + throws IOException { + // Get URL connection. + URLConnection connection = f.openConnection(); + if (connection instanceof HttpURLConnection) { + addUserAgent((HttpURLConnection)connection); + } + if (offset != STREAM_ALL) { + // Use a Range request (Assumes HTTP 1.1 on other end). If + // length >= 0, add open-ended range header to the request. Else, + // because end-byte is inclusive, subtract 1. + connection.addRequestProperty("Range", "bytes=" + offset + "-"); + } + + return getArchiveReader(f.toString(), connection.getInputStream(), + (offset == 0)); + } + + /** + * Get an ARCReader. + * Pulls the ARC local into whereever the System Property + * java.io.tmpdir points. It then hands back an ARCReader that + * points at this local copy. A close on this ARCReader instance will + * remove the local copy. + * @param u An URL that points at an ARC. + * @return An ARCReader. + * @throws IOException + */ + public static ArchiveReader get(final URL u) + throws IOException { + return ArchiveReaderFactory.factory.getArchiveReader(u); + } + + protected ArchiveReader getArchiveReader(final URL u) + throws IOException { + // If url represents a local file then return file it points to. + if (u.getPath() != null) { + // TODO: Add scheme check and host check. + File f = new File(u.getPath()); + if (f.exists()) { + return get(f, 0); + } + } + + String scheme = u.getProtocol(); + if (scheme.startsWith("http") || scheme.equals("s3")) { + // Try streaming if http or s3 URLs rather than copying local + // and then reading (Passing an offset will get us an Reader + // that wraps a Stream). + return get(u, STREAM_ALL); + } + + return makeARCLocal(u.openConnection()); + } + + protected ArchiveReader makeARCLocal(final URLConnection connection) + throws IOException { + File localFile = null; + if (connection instanceof HttpURLConnection) { + // If http url connection, bring down the resource local. + String p = connection.getURL().getPath(); + int index = p.lastIndexOf('/'); + if (index >= 0) { + // Name file for the file we're making local. + localFile = File.createTempFile("",p.substring(index + 1)); + if (localFile.exists()) { + // If file of same name already exists in TMPDIR, then + // clean it up (Assuming only reason a file of same name in + // TMPDIR is because we failed a previous download). + localFile.delete(); + } + } else { + localFile = File.createTempFile(ArchiveReader.class.getName(), + ".tmp"); + } + addUserAgent((HttpURLConnection)connection); + connection.connect(); + try { + IoUtils.readFullyToFile(connection.getInputStream(), localFile, + new byte[16 * 1024]); + } catch (IOException ioe) { + localFile.delete(); + throw ioe; + } + } else if (connection instanceof RsyncURLConnection) { + // Then, connect and this will create a local file. + // See implementation of the rsync handler. + connection.connect(); + localFile = ((RsyncURLConnection)connection).getFile(); + } else if (connection instanceof Md5URLConnection) { + // Then, connect and this will create a local file. + // See implementation of the md5 handler. + connection.connect(); + localFile = ((Md5URLConnection)connection).getFile(); + } else { + throw new UnsupportedOperationException("No support for " + + connection); + } + + ArchiveReader reader = null; + try { + reader = get(localFile, 0); + } catch (IOException e) { + localFile.delete(); + throw e; + } + + // Return a delegate that does cleanup of downloaded file on close. + return reader.getDeleteFileOnCloseReader(localFile); + } + + protected void addUserAgent(final HttpURLConnection connection) { + connection.addRequestProperty("User-Agent", this.getClass().getName()); + } + + /** + * @param f File to test. + * @return True if f is compressed. + * @throws IOException + */ + protected boolean isCompressed(final File f) throws IOException { + return f.getName().toLowerCase(). + endsWith(DOT_COMPRESSED_FILE_EXTENSION); + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/io/ArchiveRecord.java b/commons/src/main/java/org/archive/io/ArchiveRecord.java new file mode 100644 index 00000000..d1999a15 --- /dev/null +++ b/commons/src/main/java/org/archive/io/ArchiveRecord.java @@ -0,0 +1,413 @@ +/* $Id$ + * + * Created on August 21st, 2006 + * + * Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.io; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.logging.Level; + +import org.archive.util.Base32; + +/** + * Archive file Record. + * @author stack + * @version $Date$ $Version$ + */ +public abstract class ArchiveRecord extends InputStream { + + /** + * Minimal http response or request header length. + * + * I've seen in arcs content length of 1 with no header. + */ + protected static final long MIN_HTTP_HEADER_LENGTH = + Math.min("HTTP/1.1 200 OK\r\n".length(), "GET / HTTP/1.0\n\r".length()); + + ArchiveRecordHeader header = null; + + /** + * Stream to read this record from. + * + * Stream can only be read sequentially. Will only return this records' + * content returning a -1 if you try to read beyond the end of the current + * record. + * + *

Streams can be markable or not. If they are, we'll be able to roll + * back when we've read too far. If not markable, assumption is that + * the underlying stream is managing our not reading too much (This pertains + * to the skipping over the end of the ARCRecord. See {@link #skip()}. + */ + InputStream in = null; + + /** + * Position w/i the Record content, within in. + * This position is relative within this Record. Its not same as the + * Archive file position. + */ + long position = 0; + + /** + * Set flag when we've reached the end-of-record. + */ + boolean eor = false; + + /** + * Compute digest on what we read and add to metadata when done. + * + * Currently hardcoded as sha-1. TODO: Remove when archive records + * digest or else, add a facility that allows the arc reader to + * compare the calculated digest to that which is recorded in + * the arc. + * + *

Protected instead of private so subclasses can update and complete + * the digest. + */ + protected MessageDigest digest = null; + private String digestStr = null; + + boolean strict = false; + + + /** + * Constructor. + * + * @param in Stream cue'd up to be at the start of the record this instance + * is to represent. + * @throws IOException + */ + public ArchiveRecord(InputStream in) + throws IOException { + this(in, null, 0, true, false); + } + + /** + * Constructor. + * + * @param in Stream cue'd up to be at the start of the record this instance + * is to represent. + * @param header Header data. + * @throws IOException + */ + public ArchiveRecord(InputStream in, ArchiveRecordHeader header) + throws IOException { + this(in, header, 0, true, false); + } + + /** + * Constructor. + * + * @param in Stream cue'd up to be at the start of the record this instance + * is to represent. + * @param header Header data. + * @param bodyOffset Offset into the body. Usually 0. + * @param digest True if we're to calculate digest for this record. Not + * digesting saves about ~15% of cpu during an ARC parse. + * @param strict Be strict parsing (Parsing stops if ARC inproperly + * formatted). + * @throws IOException + */ + public ArchiveRecord(InputStream in, ArchiveRecordHeader header, + int bodyOffset, boolean digest, boolean strict) + throws IOException { + this.in = in; + this.header = header; + this.position = bodyOffset; + if (digest) { + try { + this.digest = MessageDigest.getInstance("SHA1"); + } catch (NoSuchAlgorithmException e) { + // Convert to IOE because thats more amenable to callers + // -- they are dealing with it anyways. + throw new IOException(e.getMessage()); + } + } + this.strict = strict; + } + + public boolean markSupported() { + return false; + } + + /** + * @return Header data for this record. + */ + public ArchiveRecordHeader getHeader() { + return this.header; + } + + protected void setHeader(ArchiveRecordHeader header) { + this.header = header; + } + + /** + * Calling close on a record skips us past this record to the next record + * in the stream. + * + * It does not actually close the stream. The underlying steam is probably + * being used by the next arc record. + * + * @throws IOException + */ + public void close() throws IOException { + if (this.in != null) { + skip(); + this.in = null; + if (this.digest != null) { + this.digestStr = Base32.encode(this.digest.digest()); + } + } + } + + /** + * @return Next character in this Record content else -1 if at EOR. + * @throws IOException + */ + public int read() throws IOException { + int c = -1; + if (available() > 0) { + c = this.in.read(); + if (c == -1) { + throw new IOException("Premature EOF before end-of-record."); + } + if (this.digest != null) { + this.digest.update((byte) c); + } + } + incrementPosition(); + return c; + } + + public int read(byte[] b, int offset, int length) throws IOException { + int read = Math.min(length, available()); + if (read == -1 || read == 0) { + read = -1; + } else { + read = this.in.read(b, offset, read); + if (read == -1) { + String msg = "Premature EOF before end-of-record: " + + getHeader().getHeaderFields(); + if (isStrict()) { + throw new IOException(msg); + } + setEor(true); + System.err.println(Level.WARNING.toString() + " " + msg); + } + if (this.digest != null && read >= 0) { + this.digest.update(b, offset, read); + } + } + incrementPosition(read); + return read; + } + + /** + * This available is not the stream's available. Its an available based on + * what the stated Archive record length is minus what we've read to date. + * + * @return True if bytes remaining in record content. + */ + public int available() { + long amount = getHeader().getLength() - getPosition(); + return (amount > Integer.MAX_VALUE? Integer.MAX_VALUE: (int)amount); + } + + /** + * Skip over this records content. + * + * @throws IOException + */ + void skip() throws IOException { + if (this.eor) { + return; + } + + // Read to the end of the body of the record. Exhaust the stream. + // Can't skip direct to end because underlying stream may be compressed + // and we're calculating the digest for the record. + int r = available(); + while (r > 0 && !this.eor) { + skip(r); + r = available(); + } + } + + public long skip(long n) throws IOException { + final int SKIP_BUFFERSIZE = 1024 * 4; + byte[] b = new byte[SKIP_BUFFERSIZE]; + long total = 0; + for (int read = 0; (total < n) && (read != -1);) { + read = Math.min(SKIP_BUFFERSIZE, (int) (n - total)); + // TODO: Interesting is that reading from compressed stream, we only + // read about 500 characters at a time though we ask for 4k. + // Look at this sometime. + read = read(b, 0, read); + if (read <= 0) { + read = -1; + } else { + total += read; + } + } + return total; + } + + /** + * @return Returns the strict. + */ + public boolean isStrict() { + return this.strict; + } + + /** + * @param strict The strict to set. + */ + public void setStrict(boolean strict) { + this.strict = strict; + } + + protected InputStream getIn() { + return this.in; + } + + public String getDigestStr() { + return this.digestStr; + } + + protected void incrementPosition() { + this.position++; + } + + protected void incrementPosition(final long incr) { + this.position += incr; + } + + protected long getPosition() { + return this.position; + } + + protected boolean isEor() { + return eor; + } + + protected void setEor(boolean eor) { + this.eor = eor; + } + + protected String getStatusCode4Cdx(final ArchiveRecordHeader h) { + return "-"; + } + + protected String getIp4Cdx(final ArchiveRecordHeader h) { + return "-"; + } + + protected String getDigest4Cdx(final ArchiveRecordHeader h) { + return getDigestStr() == null? "-": getDigestStr(); + } + + protected String getMimetype4Cdx(final ArchiveRecordHeader h) { + return h.getMimetype(); + } + + protected String outputCdx(final String strippedFileName) + throws IOException { + // Read the whole record so we get out a hash. Should be safe calling + // close on already closed Record. + close(); + ArchiveRecordHeader h = getHeader(); + StringBuilder buffer = + new StringBuilder(ArchiveFileConstants.CDX_LINE_BUFFER_SIZE); + buffer.append(h.getDate()); + buffer.append(ArchiveFileConstants.SINGLE_SPACE); + buffer.append(getIp4Cdx(h)); + buffer.append(ArchiveFileConstants.SINGLE_SPACE); + buffer.append(h.getUrl()); + buffer.append(ArchiveFileConstants.SINGLE_SPACE); + buffer.append(getMimetype4Cdx(h)); + buffer.append(ArchiveFileConstants.SINGLE_SPACE); + buffer.append(getStatusCode4Cdx(h)); + buffer.append(ArchiveFileConstants.SINGLE_SPACE); + buffer.append(getDigest4Cdx(h)); + buffer.append(ArchiveFileConstants.SINGLE_SPACE); + buffer.append(h.getOffset()); + buffer.append(ArchiveFileConstants.SINGLE_SPACE); + buffer.append(h.getLength()); + buffer.append(ArchiveFileConstants.SINGLE_SPACE); + buffer.append(strippedFileName != null? strippedFileName: '-'); + return buffer.toString(); + } + + /** + * Writes output on STDOUT. + * @throws IOException + */ + public void dump() + throws IOException { + dump(System.out); + } + + /** + * Writes output on passed os. + * @throws IOException + */ + public void dump(final OutputStream os) + throws IOException { + final byte [] outputBuffer = new byte [16*1024]; + int read = outputBuffer.length; + while ((read = read(outputBuffer, 0, outputBuffer.length)) != -1) { + os.write(outputBuffer, 0, read); + } + os.flush(); + } + + /** + * Is it likely that this record contains headers? + * This method will return true if the body is a http response that includes + * http response headers or the body is a http request that includes request + * headers, etc. Be aware that headers in content are distinct from + * {@link ArchiveRecordHeader} 'headers'. + * @return True if this Record's content has headers: + */ + public boolean hasContentHeaders() { + final String url = getHeader().getUrl(); + if (url == null) { + return false; + } + + if (!url.toLowerCase().startsWith("http")) { + return false; + } + + if (getHeader().getLength() <= MIN_HTTP_HEADER_LENGTH) { + return false; + } + + return true; + } + + protected void setBodyOffset(int bodyOffset) { + this.position = bodyOffset; + } +} diff --git a/commons/src/main/java/org/archive/io/ArchiveRecordHeader.java b/commons/src/main/java/org/archive/io/ArchiveRecordHeader.java new file mode 100644 index 00000000..2a524feb --- /dev/null +++ b/commons/src/main/java/org/archive/io/ArchiveRecordHeader.java @@ -0,0 +1,109 @@ +/* $Id$ + * + * Created on August 21st, 2006 + * + * Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.io; + +import java.util.Map; +import java.util.Set; + +/** + * Archive Record Header. + * @author stack + * @version $Date$ $Version$ + */ +public interface ArchiveRecordHeader { + /** + * Get the time when the record was created. + * @return Date in 14 digit time format (UTC). + * @see org.archive.util.ArchiveUtils#parse14DigitDate(String) + */ + public abstract String getDate(); + + /** + * @return Return length of record. + */ + public abstract long getLength(); + + /** + * @return Record subject-url. + */ + public abstract String getUrl(); + + /** + * @return Record mimetype. + */ + public abstract String getMimetype(); + + /** + * @return Record version. + */ + public abstract String getVersion(); + + /** + * @return Offset into Archive file at which this record begins. + */ + public abstract long getOffset(); + + /** + * @param key Key to use looking up field value. + * @return value for passed key of null if no such entry. + */ + public abstract Object getHeaderValue(final String key); + + /** + * @return Header field name keys. + */ + public abstract Set getHeaderFieldKeys(); + + /** + * @return Map of header fields. + */ + public abstract Map getHeaderFields(); + + /** + * @return Returns identifier for current Archive file. Be aware this + * may not be a file name or file path. It may just be an URL. Depends + * on how Archive file was made. + */ + public abstract String getReaderIdentifier(); + + /** + * @return Identifier for the record. If ARC, the URL + date. If WARC, + * the GUID assigned. + */ + public abstract String getRecordIdentifier(); + + /** + * @return Returns digest as String for this record. Only available after + * the record has been read in totality. + */ + public abstract String getDigest(); + + /** + * Offset at which the content begins. + * For ARCs, its used to delimit where http headers end and content begins. + * For WARCs, its end of Named Fields before payload starts. + */ + public int getContentBegin(); + + public abstract String toString(); +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/io/ArraySeekInputStream.java b/commons/src/main/java/org/archive/io/ArraySeekInputStream.java new file mode 100644 index 00000000..8b9df9c3 --- /dev/null +++ b/commons/src/main/java/org/archive/io/ArraySeekInputStream.java @@ -0,0 +1,110 @@ +/* ArraySeekInputStream +* +* Created on September 18, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.io; + +import java.io.IOException; + + +/** + * A repositionable stream backed by an array. + * + * @author pjack + */ +public class ArraySeekInputStream extends SeekInputStream { + + + /** + * The array of bytes to read from. + */ + private byte[] array; + + + /** + * The offset in the array of the next byte to read. + */ + private int offset; + + + /** + * Constructor. Note that changes to the given array will be reflected + * in the stream. + * + * @param array The array to read bytes from. + */ + public ArraySeekInputStream(byte[] array) { + this.array = array; + this.offset = 0; + } + + + @Override + public int read() { + if (offset >= array.length) { + return -1; + } + int r = array[offset] & 0xFF; + offset++; + return r; + } + + + @Override + public int read(byte[] buf, int ofs, int len) { + if (offset >= array.length) { + return 0; + } + len = Math.min(len, array.length - offset); + System.arraycopy(array, offset, buf, ofs, len); + offset += len; + return len; + } + + + @Override + public int read(byte[] buf) { + return read(buf, 0, buf.length); + } + + + /** + * Returns the position of the stream. + */ + public long position() { + return offset; + } + + + /** + * Repositions the stream. + * + * @param p the new position for the stream + * @throws IOException if the given position is out of bounds + */ + public void position(long p) throws IOException { + if ((p < 0) || (p > array.length)) { + throw new IOException("Invalid position: " + p); + } + offset = (int)p; + } + +} diff --git a/commons/src/main/java/org/archive/io/BufferedSeekInputStream.java b/commons/src/main/java/org/archive/io/BufferedSeekInputStream.java new file mode 100644 index 00000000..ea700884 --- /dev/null +++ b/commons/src/main/java/org/archive/io/BufferedSeekInputStream.java @@ -0,0 +1,221 @@ +/* BufferedSeekInputStream +* +* Created on September 14, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.io; + + +import java.io.IOException; + + +/** + * Buffers data from some other SeekInputStream. + * + * @author pjack + */ +public class BufferedSeekInputStream extends SeekInputStream { + + + /** + * The underlying input stream. + */ + final private SeekInputStream input; + + + /** + * The buffered data. + */ + final private byte[] buffer; + + + /** + * The maximum offset of valid data in the buffer. Usually the same + * as buffer.length, but may be shorter if we're in the last region + * of the stream. + */ + private int maxOffset; + + + /** + * The offset of within the buffer of the next byte to read. + */ + private int offset; + + + /** + * Constructor. + * + * @param input the underlying input stream + * @param capacity the size of the buffer + * @throws IOException if an IO occurs filling the first buffer + */ + public BufferedSeekInputStream(SeekInputStream input, int capacity) + throws IOException { + this.input = input; + this.buffer = new byte[capacity]; + buffer(); + } + + /** + * Fills the buffer. + * + * @throws IOException if an IO error occurs + */ + private void buffer() throws IOException { + int remaining = buffer.length; + while (remaining > 0) { + int r = input.read(buffer, buffer.length - remaining, remaining); + if (r <= 0) { + // Not enough information to fill the buffer + offset = 0; + maxOffset = buffer.length - remaining; + return; + } + remaining -= r; + } + maxOffset = buffer.length; + offset = 0; + } + + + /** + * Ensures that the buffer is valid. + * + * @throws IOException if an IO error occurs + */ + private void ensureBuffer() throws IOException { + if (offset >= maxOffset) { + buffer(); + } + } + + + /** + * Returns the number of unread bytes in the current buffer. + * + * @return the remaining bytes + */ + private int remaining() { + return maxOffset - offset; + } + + + @Override + public int read() throws IOException { + ensureBuffer(); + if (maxOffset == 0) { + return -1; + } + int ch = buffer[offset] & 0xFF; + offset++; + return ch; + } + + + @Override + public int read(byte[] buf, int ofs, int len) throws IOException { + ensureBuffer(); + if (maxOffset == 0) { + return 0; + } + len = Math.min(len, remaining()); + System.arraycopy(buffer, offset, buf, ofs, len); + offset += len; + return len; + } + + + @Override + public int read(byte[] buf) throws IOException { + return read(buf, 0, buf.length); + } + + + @Override + public long skip(long c) throws IOException { + ensureBuffer(); + if (maxOffset == 0) { + return 0; + } + int count = (c > Integer.MAX_VALUE) ? Integer.MAX_VALUE : (int)c; + int skip = Math.min(count, remaining()); + offset += skip; + return skip; + } + + + /** + * Returns the stream's current position. + * + * @return the current position + */ + public long position() throws IOException { + return input.position() - buffer.length + offset; + } + + + /** + * Seeks to the given position. This method avoids re-filling the buffer + * if at all possible. + * + * @param p the position to set + * @throws IOException if an IO error occurs + */ + public void position(long p) throws IOException { + long blockStart = (input.position() - maxOffset) + / buffer.length * buffer.length; + long blockEnd = blockStart + maxOffset; + if ((p >= blockStart) && (p < blockEnd)) { + // Desired position is somewhere inside current buffer + long adj = p - blockStart; + offset = (int)adj; + return; + } + positionDirect(p); + } + + + /** + * Positions the underlying stream at the given position, then refills + * the buffer. + * + * @param p the position to set + * @throws IOException if an IO error occurs + */ + private void positionDirect(long p) throws IOException { + long newBlockStart = p / buffer.length * buffer.length; + input.position(newBlockStart); + buffer(); + offset = (int)(p % buffer.length); + } + + /** + * Close the stream, including the wrapped input stream. + */ + public void close() throws IOException { + super.close(); + if(this.input!=null) { + this.input.close(); + } + } + + +} diff --git a/commons/src/main/java/org/archive/io/ByteReplayCharSequence.java b/commons/src/main/java/org/archive/io/ByteReplayCharSequence.java new file mode 100644 index 00000000..6d57ef48 --- /dev/null +++ b/commons/src/main/java/org/archive/io/ByteReplayCharSequence.java @@ -0,0 +1,363 @@ +/* ByteReplayCharSequenceFactory + * + * (Re)Created on Dec 21, 2006 + * + * Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.io; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.archive.util.DevUtils; + +/** + * Provides a (Replay)CharSequence view on recorded stream bytes (a prefix + * buffer and overflow backing file). + * + * Treats the byte stream as 8-bit. + * + *

Uses a wraparound rolling buffer of the last windowSize bytes read + * from disk in memory; as long as the 'random access' of a CharSequence + * user stays within this window, access should remain fairly efficient. + * (So design any regexps pointed at these CharSequences to work within + * that range!) + * + *

When rereading of a location is necessary, the whole window is + * recentered around the location requested. (TODO: More research + * into whether this is the best strategy.) + * + *

An implementation of a ReplayCharSequence done with ByteBuffers -- one + * to wrap the passed prefix buffer and the second, a memory-mapped + * ByteBuffer view into the backing file -- was consistently slower: ~10%. + * My tests did the following. Made a buffer filled w/ regular content. + * This buffer was used as the prefix buffer. The buffer content was + * written MULTIPLER times to a backing file. I then did accesses w/ the + * following pattern: Skip forward 32 bytes, then back 16 bytes, and then + * read forward from byte 16-32. Repeat. Though I varied the size of the + * buffer to the size of the backing file,from 3-10, the difference of 10% + * or so seemed to persist. Same if I tried to favor get() over get(index). + * I used a profiler, JMP, to study times taken (St.Ack did above comment). + * + *

TODO determine in memory mapped files is better way to do this; + * probably not -- they don't offer the level of control over + * total memory used that this approach does. + * + * @author Gordon Mohr + * @version $Revision$, $Date$ + */ +class ByteReplayCharSequence implements ReplayCharSequence { + + protected static Logger logger = + Logger.getLogger(ByteReplayCharSequence.class.getName()); + + /** + * Buffer that holds the first bit of content. + * + * Once this is exhausted we go to the backing file. + */ + private byte[] prefixBuffer; + + /** + * Total length of character stream to replay minus the HTTP headers + * if present. + * + * Used to find EOS. + */ + protected int length; + + /** + * Absolute length of the stream. + * + * Includes HTTP headers. Needed doing calc. in the below figuring + * how much to load into buffer. + */ + private int absoluteLength = -1; + + /** + * Buffer window on to backing file. + */ + private byte[] wraparoundBuffer; + + /** + * Absolute index into underlying bytestream where wrap starts. + */ + private int wrapOrigin; + + /** + * Index in wraparoundBuffer that corresponds to wrapOrigin + */ + private int wrapOffset; + + /** + * Name of backing file we go to when we've exhausted content from the + * prefix buffer. + */ + private String backingFilename; + + /** + * Random access to the backing file. + */ + private RandomAccessFile raFile; + + /** + * Offset into prefix buffer at which content beings. + */ + private int contentOffset; + + /** + * 8-bit encoding used reading single bytes from buffer and + * stream. + */ + @SuppressWarnings("unused") + private static final String DEFAULT_SINGLE_BYTE_ENCODING = + "ISO-8859-1"; + + + /** + * Constructor. + * + * @param buffer In-memory buffer of recordings prefix. We read from + * here first and will only go to the backing file if size + * requested is greater than buffer.length. + * @param size Total size of stream to replay in bytes. Used to find + * EOS. This is total length of content including HTTP headers if + * present. + * @param responseBodyStart Where the response body starts in bytes. + * Used to skip over the HTTP headers if present. + * @param backingFilename Path to backing file with content in excess of + * whats in buffer. + * + * @throws IOException + */ + public ByteReplayCharSequence(byte[] buffer, long size, + long responseBodyStart, String backingFilename) + throws IOException { + + this.length = (int)(size - responseBodyStart); + this.absoluteLength = (int)size; + this.prefixBuffer = buffer; + this.contentOffset = (int)responseBodyStart; + + // If amount to read is > than what is in our prefix buffer, then + // open the backing file. + if (size > buffer.length) { + this.backingFilename = backingFilename; + this.raFile = new RandomAccessFile(backingFilename, "r"); + this.wraparoundBuffer = new byte[this.prefixBuffer.length]; + this.wrapOrigin = this.prefixBuffer.length; + this.wrapOffset = 0; + loadBuffer(); + } + } + + /** + * @return Length of characters in stream to replay. Starts counting + * at the HTTP header/body boundary. + */ + public int length() { + return this.length; + } + + /** + * Get character at passed absolute position. + * + * Called by {@link #charAt(int)} which has a relative index into the + * content, one that doesn't account for HTTP header if present. + * + * @param index Index into content adjusted to accomodate initial offset + * to get us past the HTTP header if present (i.e. + * {@link #contentOffset}). + * + * @return Characater at offset index. + */ + public char charAt(int index) { + int c = -1; + // Add to index start-of-content offset to get us over HTTP header + // if present. + index += this.contentOffset; + if (index < this.prefixBuffer.length) { + // If index is into our prefix buffer. + c = this.prefixBuffer[index]; + } else if (index >= this.wrapOrigin && + (index - this.wrapOrigin) < this.wraparoundBuffer.length) { + // If index is into our buffer window on underlying backing file. + c = this.wraparoundBuffer[ + ((index - this.wrapOrigin) + this.wrapOffset) % + this.wraparoundBuffer.length]; + } else { + // Index is outside of both prefix buffer and our buffer window + // onto the underlying backing file. Fix the buffer window + // location. + c = faultCharAt(index); + } + // Stream is treated as single byte. Make sure characters returned + // are not negative. + return (char)(c & 0xff); + } + + /** + * Get a character that's outside the current buffers. + * + * will cause the wraparoundBuffer to be changed to + * cover a region including the index + * + * if index is higher than the highest index in the + * wraparound buffer, buffer is moved forward such + * that requested char is last item in buffer + * + * if index is lower than lowest index in the + * wraparound buffer, buffet is reset centered around + * index + * + * @param index Index of character to fetch. + * @return A character that's outside the current buffers + */ + private int faultCharAt(int index) { + if(Thread.interrupted()) { + throw new RuntimeException("thread interrupted"); + } + if(index >= this.wrapOrigin + this.wraparoundBuffer.length) { + // Moving forward + while (index >= this.wrapOrigin + this.wraparoundBuffer.length) + { + // TODO optimize this + advanceBuffer(); + } + return charAt(index - this.contentOffset); + } + // Moving backward + recenterBuffer(index); + return charAt(index - this.contentOffset); + } + + /** + * Move the buffer window on backing file back centering current access + * position in middle of window. + * + * @param index Index of character to access. + */ + private void recenterBuffer(int index) { + if (logger.isLoggable(Level.FINE)) { + logger.fine("Recentering around " + index + " in " + + this.backingFilename); + } + this.wrapOrigin = index - (this.wraparoundBuffer.length / 2); + if(this.wrapOrigin < this.prefixBuffer.length) { + this.wrapOrigin = this.prefixBuffer.length; + } + this.wrapOffset = 0; + loadBuffer(); + } + + /** + * Load from backing file into the wrapper buffer. + */ + private void loadBuffer() + { + long len = -1; + try { + len = this.raFile.length(); + this.raFile.seek(this.wrapOrigin - this.prefixBuffer.length); + this.raFile.readFully(this.wraparoundBuffer, 0, + Math.min(this.wraparoundBuffer.length, + this.absoluteLength - this.wrapOrigin)); + } + + catch (IOException e) { + // TODO convert this to a runtime error? + DevUtils.logger.log ( + Level.SEVERE, + "raFile.seek(" + + (this.wrapOrigin - this.prefixBuffer.length) + + ")\n" + + "raFile.readFully(wraparoundBuffer,0," + + (Math.min(this.wraparoundBuffer.length, + this.length - this.wrapOrigin )) + + ")\n"+ + "raFile.length()" + len + "\n" + + DevUtils.extraInfo(), + e); + throw new RuntimeException(e); + } + } + + /** + * Roll the wraparound buffer forward one position + */ + private void advanceBuffer() { + try { + this.wraparoundBuffer[this.wrapOffset] = + (byte)this.raFile.read(); + this.wrapOffset++; + this.wrapOffset %= this.wraparoundBuffer.length; + this.wrapOrigin++; + } catch (IOException e) { + DevUtils.logger.log(Level.SEVERE, "advanceBuffer()" + + DevUtils.extraInfo(), e); + throw new RuntimeException(e); + } + } + + public CharSequence subSequence(int start, int end) { + return new CharSubSequence(this, start, end); + } + + /** + * Cleanup resources. + * + * @exception IOException Failed close of random access file. + */ + public void close() throws IOException + { + this.prefixBuffer = null; + if (this.raFile != null) { + this.raFile.close(); + this.raFile = null; + } + } + + /* (non-Javadoc) + * @see java.lang.Object#finalize() + */ + protected void finalize() throws Throwable + { + super.finalize(); + close(); + } + + /** + * Convenience method for getting a substring. + * @deprecated please use subSequence() and then toString() directly + */ + public String substring(int offset, int len) { + return subSequence(offset, offset+len).toString(); + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + public String toString() { + StringBuilder sb = new StringBuilder(this.length()); + sb.append(this); + return sb.toString(); + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/io/CharSubSequence.java b/commons/src/main/java/org/archive/io/CharSubSequence.java new file mode 100644 index 00000000..8b22dfe1 --- /dev/null +++ b/commons/src/main/java/org/archive/io/CharSubSequence.java @@ -0,0 +1,94 @@ +/* CharSubSequence.java + * + * Created on Sep 30, 2003 + * + * Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.io; + + +/** + * Provides a subsequence view onto a CharSequence. + * + * @author gojomo + * @version $Revision$, $Date$ + */ +public class CharSubSequence implements CharSequence { + + CharSequence inner; + int start; + int end; + + public CharSubSequence(CharSequence inner, int start, int end) { + if (end < start) { + throw new IllegalArgumentException("Start " + start + " is > " + + " than end " + end); + } + + if (end < 0 || start < 0) { + throw new IllegalArgumentException("Start " + start + " or end " + + end + " is < 0."); + } + + if (inner == null) { + throw new NullPointerException("Passed charsequence is null."); + } + + this.inner = inner; + this.start = start; + this.end = end; + } + + /* + * (non-Javadoc) + * @see java.lang.CharSequence#length() + */ + public int length() { + return this.end - this.start; + } + + /* + * (non-Javadoc) + * @see java.lang.CharSequence#charAt(int) + */ + public char charAt(int index) { + return this.inner.charAt(this.start + index); + } + + /* + * (non-Javadoc) + * @see java.lang.CharSequence#subSequence(int, int) + */ + public CharSequence subSequence(int begin, int finish) { + return new CharSubSequence(this, begin, finish); + } + + /* + * (non-Javadoc) + * @see java.lang.CharSequence#toString() + */ + public String toString() { + StringBuffer sb = new StringBuffer(length()); + // could use StringBuffer.append(CharSequence) if willing to do 1.5 & up + for (int i = 0;i filenames; + + /* (non-Javadoc) + * @see java.io.InputStream#read() + */ + public int read() throws IOException { + int c = super.read(); + if( c == -1 && filenames.hasNext() ) { + cueStream(); + return read(); + } + return c; + } + /* (non-Javadoc) + * @see java.io.InputStream#read(byte[], int, int) + */ + public int read(byte[] b, int off, int len) throws IOException { + int c = super.read(b, off, len); + if( c == -1 && filenames.hasNext() ) { + cueStream(); + return read(b,off,len); + } + return c; + } + /* (non-Javadoc) + * @see java.io.InputStream#read(byte[]) + */ + public int read(byte[] b) throws IOException { + int c = super.read(b); + if( c == -1 && filenames.hasNext() ) { + cueStream(); + return read(b); + } + return c; + } + + /* (non-Javadoc) + * @see java.io.InputStream#skip(long) + */ + public long skip(long n) throws IOException { + long s = super.skip(n); + if( s files) throws IOException { + super(null); + filenames = files.iterator(); + cueStream(); + } + + private void cueStream() throws IOException { + if(filenames.hasNext()) { + this.in = new FileInputStream(filenames.next()); + } + } + +} diff --git a/commons/src/main/java/org/archive/io/CompositeFileReader.java b/commons/src/main/java/org/archive/io/CompositeFileReader.java new file mode 100644 index 00000000..ff1e0fa1 --- /dev/null +++ b/commons/src/main/java/org/archive/io/CompositeFileReader.java @@ -0,0 +1,46 @@ +/* CompositeFileReader +* +* $Id$ +* +* Created on May 18, 2004 +* +* Copyright (C) 2004 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.io; + +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.List; + + +/** + * @author gojomo + */ +public class CompositeFileReader extends InputStreamReader { + + /** + * @param filenames + * @throws IOException + */ + public CompositeFileReader(List filenames) throws IOException { + super(new CompositeFileInputStream(filenames)); + } + +} diff --git a/commons/src/main/java/org/archive/io/CrawlerJournal.java b/commons/src/main/java/org/archive/io/CrawlerJournal.java new file mode 100644 index 00000000..ee595ad3 --- /dev/null +++ b/commons/src/main/java/org/archive/io/CrawlerJournal.java @@ -0,0 +1,271 @@ +/* CrawlerJournal.java + * + * Created on Mar 6, 2007 + * + * Copyright (C) 2007 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.io; + +import it.unimi.dsi.fastutil.io.FastBufferedOutputStream; +import it.unimi.dsi.mg4j.util.MutableString; + +import java.io.BufferedInputStream; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.util.List; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +import org.archive.checkpointing.RecoverAction; +import org.archive.util.ArchiveUtils; +import org.archive.util.FileUtils; + +/** + * Utility class for a crawler journal/log that is compressed and + * rotates by serial number at checkpoints. + * + * @author gojomo + */ +public class CrawlerJournal { + + /** prefix for error lines*/ + public static final String LOG_ERROR = "E "; + /** prefix for timestamp lines */ + public static final String LOG_TIMESTAMP = "T "; + + /** + * Get a BufferedReader on the crawler journal given + * + * TODO: move to a general utils class + * + * @param source File journal + * @return journal buffered reader. + * @throws IOException + */ + public static BufferedReader getBufferedReader(File source) throws IOException { + InputStream is = new BufferedInputStream(new FileInputStream(source)); + boolean isGzipped = source.getName().toLowerCase(). + endsWith(GZIP_SUFFIX); + if(isGzipped) { + is = new GZIPInputStream(is); + } + return new BufferedReader(new InputStreamReader(is)); + } + + /** + * Stream on which we record frontier events. + */ + protected Writer out = null; + + /** line count */ + protected long lines = 0; + /** number of lines between timestamps */ + protected int timestamp_interval = 0; // 0 means no timestamps + + + /** suffix to recognize gzipped files */ + public static final String GZIP_SUFFIX = ".gz"; + + /** + * File we're writing journal to. + * Keep a reference in case we want to rotate it off. + */ + protected File gzipFile = null; + + /** + * Create a new crawler journal at the given location + * + * @param path Directory to make thejournal in. + * @param filename Name to use for journal file. + * @throws IOException + */ + public CrawlerJournal(String path, String filename) + throws IOException { + this.gzipFile = new File(path, filename); + this.out = initialize(gzipFile); + } + + /** + * Create a new crawler journal at the given location + * + * @param file path at which to make journal + * @throws IOException + */ + public CrawlerJournal(File file) throws IOException { + this.gzipFile = file; + this.out = initialize(gzipFile); + } + + /** + * Allocate a buffer for accumulating lines to write and reuse it. + */ + protected MutableString accumulatingBuffer = new MutableString(1024); + + protected Writer initialize(final File f) throws FileNotFoundException, IOException { + FileUtils.moveAsideIfExists(f); + return new OutputStreamWriter(new GZIPOutputStream( + new FastBufferedOutputStream(new FileOutputStream(f)))); + } + + /** + * Write a line + * + * @param string String + */ + public synchronized void writeLine(String string) { + try { + this.out.write(string); + this.out.write("\n"); + noteLine(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + /** + * Write a line of two strings + * + * @param s1 String + * @param s2 String + */ + public synchronized void writeLine(String s1, String s2) { + try { + this.out.write(s1); + this.out.write(s2); + this.out.write("\n"); + noteLine(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + /** + * Write a line of three strings + * + * @param s1 String + * @param s2 String + * @param s3 String + */ + public synchronized void writeLine(String s1, String s2, String s3) { + try { + this.out.write(s1); + this.out.write(s2); + this.out.write(s3); + this.out.write("\n"); + noteLine(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + /** + * Write a line. + * + * @param mstring MutableString to write + */ + public synchronized void writeLine(MutableString mstring) { + if (this.out == null) { + return; + } + try { + mstring.write(out); + this.out.write("\n"); + noteLine(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + /** + * Count and note a line + * + * @throws IOException + */ + protected void noteLine() throws IOException { + lines++; + considerTimestamp(); + } + + /** + * Write a timestamp line if appropriate + * + * @throws IOException + */ + protected void considerTimestamp() throws IOException { + if(timestamp_interval > 0 && lines % timestamp_interval == 0) { + out.write(LOG_TIMESTAMP); + out.write(ArchiveUtils.getLog14Date()); + out.write("\n"); + } + } + + /** + * Flush and close the underlying IO objects. + */ + public void close() { + if (this.out == null) { + return; + } + try { + this.out.flush(); + this.out.close(); + this.out = null; + } catch (IOException e) { + e.printStackTrace(); + } + } + + /** + * Note a serious error vioa a special log line + * + * @param err + */ + public synchronized void seriousError(String err) { + writeLine(LOG_ERROR+ArchiveUtils.getLog14Date()+" "+err+"\n"); + } + + /** + * Handle a checkpoint by rotating the current log to a checkpoint-named + * file and starting a new log. + * + * @param checkpointDir + * @throws IOException + */ + public synchronized void checkpoint(final File checkpointDir, + List actions) throws IOException { + if (this.out == null || !this.gzipFile.exists()) { + return; + } + close(); + // Rename gzipFile with the checkpoint name as suffix. + this.gzipFile.renameTo(new File(this.gzipFile.getParentFile(), + this.gzipFile.getName() + "." + checkpointDir.getName())); + // Open new gzip file. + this.out = initialize(this.gzipFile); + } + +} diff --git a/commons/src/main/java/org/archive/io/DefaultWriterPoolSettings.java b/commons/src/main/java/org/archive/io/DefaultWriterPoolSettings.java new file mode 100644 index 00000000..2dcaf53d --- /dev/null +++ b/commons/src/main/java/org/archive/io/DefaultWriterPoolSettings.java @@ -0,0 +1,150 @@ +/* + * Copyright (C) 2007 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * DefaultWriterPoolSettings.java + * + * Created on Mar 11, 2007 + * + * $Id:$ + */ + +package org.archive.io; + +import java.io.File; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import org.archive.checkpointing.CheckpointRecovery; + +/** + * @author pjack + * + */ +public class DefaultWriterPoolSettings +implements WriterPoolSettings, Serializable { + + + private static final long serialVersionUID = 1L; + + + private long maxSize; + private List metadata = new ArrayList();; + transient private List outputDirs = new ArrayList(); + private String prefix; + private String suffix; + private boolean compressed; + + + public DefaultWriterPoolSettings() { + } + + + public boolean isCompressed() { + return compressed; + } + + + public void setCompressed(boolean compressed) { + this.compressed = compressed; + } + + + public long getMaxSize() { + return maxSize; + } + + + public void setMaxSize(long maxSize) { + this.maxSize = maxSize; + } + + + public List getMetadata() { + return metadata; + } + + + public void setMetadata(List metadata) { + this.metadata = metadata; + } + + + public List getOutputDirs() { + return outputDirs; + } + + + public void setOutputDirs(List outputDirs) { + this.outputDirs = outputDirs; + } + + + public String getPrefix() { + return prefix; + } + + + public void setPrefix(String prefix) { + this.prefix = prefix; + } + + + public String getSuffix() { + return suffix; + } + + + public void setSuffix(String suffix) { + this.suffix = suffix; + } + + + private void writeObject(ObjectOutputStream out) throws IOException { + out.defaultWriteObject(); + out.writeInt(outputDirs.size()); + for (File f: outputDirs) { + out.writeUTF(f.getAbsolutePath()); + } + } + + + private void readObject(ObjectInputStream input) + throws IOException, ClassNotFoundException { + input.defaultReadObject(); + CheckpointRecovery cr = null; + this.outputDirs = new ArrayList(); + if (input instanceof CheckpointRecovery) { + cr = (CheckpointRecovery)input; + } + int size = input.readInt(); + for (int i = 0; i < size; i++) { + String path = input.readUTF(); + if (cr != null) { + path = cr.translatePath(path); + } + File f = new File(path); + f.mkdirs(); + outputDirs.add(f); + } + } +} diff --git a/commons/src/main/java/org/archive/io/Endian.java b/commons/src/main/java/org/archive/io/Endian.java new file mode 100644 index 00000000..55121429 --- /dev/null +++ b/commons/src/main/java/org/archive/io/Endian.java @@ -0,0 +1,129 @@ +/* Endian +* +* Created on September 12, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.io; + + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; + + +/** + * Reads integers stored in big or little endian streams. + * + * @author pjack + */ +public class Endian { + + + /** + * Static utility class. + */ + private Endian() { + } + + + /** + * Reads the next little-endian unsigned 16 bit integer from the + * given stream. + * + * @param input the input stream to read from + * @return the next 16-bit little-endian integer + * @throws IOException if an IO error occurs + */ + public static char littleChar(InputStream input) throws IOException { + int lo = input.read(); + if (lo < 0) { + throw new EOFException(); + } + int hi = input.read(); + if (hi < 0) { + throw new EOFException(); + } + return (char)((hi << 8) | lo); + } + + + /** + * Reads the next little-endian signed 16-bit integer from the + * given stream. + * + * @param input the input stream to read from + * @return the next 16-bit little-endian integer + * @throws IOException if an IO error occurs + */ + public static short littleShort(InputStream input) throws IOException { + return (short)littleChar(input); + } + + + /** + * Reads the next little-endian signed 32-bit integer from the + * given stream. + * + * @param input the input stream to read from + * @return the next 32-bit little-endian integer + * @throws IOException if an IO error occurs + */ + public static int littleInt(InputStream input) throws IOException { + char lo = littleChar(input); + char hi = littleChar(input); + return (hi << 16) | lo; + } + + + /** + * Reads the next big-endian unsigned 16 bit integer from the + * given stream. + * + * @param input the input stream to read from + * @return the next 16-bit big-endian integer + * @throws IOException if an IO error occurs + */ + public static char bigChar(InputStream input) throws IOException { + int hi = input.read(); + if (hi < 0) { + throw new EOFException(); + } + int lo = input.read(); + if (lo < 0) { + throw new EOFException(); + } + return (char)((hi << 8) | lo); + } + + + /** + * Reads the next big-endian signed 32-bit integer from the + * given stream. + * + * @param input the input stream to read from + * @return the next 32-bit big-endian integer + * @throws IOException if an IO error occurs + */ + public static int bigInt(InputStream input) throws IOException { + char hi = bigChar(input); + char lo = bigChar(input); + return (hi << 16) | lo; + } +} diff --git a/commons/src/main/java/org/archive/io/GenerationFileHandler.java b/commons/src/main/java/org/archive/io/GenerationFileHandler.java new file mode 100644 index 00000000..a8cb5ac7 --- /dev/null +++ b/commons/src/main/java/org/archive/io/GenerationFileHandler.java @@ -0,0 +1,136 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.logging.FileHandler; + +import org.archive.util.FileUtils; + + +/** + * FileHandler with support for rotating the current file to + * an archival name with a specified integer suffix, and + * provision of a new replacement FileHandler with the current + * filename. + * + * @author gojomo + */ +public class GenerationFileHandler extends FileHandler { + private LinkedList filenameSeries = new LinkedList(); + private boolean shouldManifest = false; + + /** + * @return Returns the filenameSeries. + */ + public List getFilenameSeries() { + return filenameSeries; + } + + /** + * Constructor. + * @param pattern + * @param append + * @param shouldManifest + * @throws IOException + * @throws SecurityException + */ + public GenerationFileHandler(String pattern, boolean append, + boolean shouldManifest) + throws IOException, SecurityException { + super(pattern, append); + filenameSeries.addFirst(pattern); + this.shouldManifest = shouldManifest; + } + + /** + * @param filenameSeries + * @param shouldManifest + * @throws IOException + */ + public GenerationFileHandler(LinkedList filenameSeries, + boolean shouldManifest) + throws IOException { + super((String)filenameSeries.getFirst(), false); // Never append in this case + this.filenameSeries = filenameSeries; + this.shouldManifest = shouldManifest; + } + + /** + * Move the current file to a new filename with the storeSuffix in place + * of the activeSuffix; continuing logging to a new file under the + * original filename. + * + * @param storeSuffix Suffix to put in place of activeSuffix + * @param activeSuffix Suffix to replace with storeSuffix. + * @return GenerationFileHandler instance. + * @throws IOException + */ + public GenerationFileHandler rotate(String storeSuffix, + String activeSuffix) + throws IOException { + close(); + String filename = (String)filenameSeries.getFirst(); + if (!filename.endsWith(activeSuffix)) { + throw new FileNotFoundException("Active file does not have" + + " expected suffix"); + } + String storeFilename = filename.substring(0, + filename.length() - activeSuffix.length()) + + storeSuffix; + File activeFile = new File(filename); + File storeFile = new File(storeFilename); + if (!activeFile.renameTo(storeFile)) { + throw new IOException("Unable to move " + filename + " to " + + storeFilename); + } + filenameSeries.add(1, storeFilename); + GenerationFileHandler newGfh = + new GenerationFileHandler(filenameSeries, shouldManifest); + newGfh.setFormatter(this.getFormatter()); + return newGfh; + } + + /** + * @return True if should manifest. + */ + public boolean shouldManifest() { + return this.shouldManifest; + } + + /** + * Constructor-helper that rather than clobbering any existing + * file, moves it aside with a timestamp suffix. + * + * @param filename + * @param append + * @param shouldManifest + * @return + * @throws SecurityException + * @throws IOException + */ + public static GenerationFileHandler makeNew(String filename, boolean append, boolean shouldManifest) throws SecurityException, IOException { + FileUtils.moveAsideIfExists(new File(filename)); + return new GenerationFileHandler(filename, append, shouldManifest); + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/io/GzipHeader.java b/commons/src/main/java/org/archive/io/GzipHeader.java new file mode 100644 index 00000000..731a8791 --- /dev/null +++ b/commons/src/main/java/org/archive/io/GzipHeader.java @@ -0,0 +1,344 @@ +/* GzipHeader +* +* $Id$ +* +* Created on July 5, 2004 +* +* Copyright (C) 2004 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.io; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.util.zip.CRC32; +import java.util.zip.Deflater; +import java.util.zip.GZIPInputStream; + +/** + * Read in the GZIP header. + * + * See RFC1952 for specification on what the header looks like. + * Assumption is that stream is cued-up with the gzip header as the + * next thing to be read. + * + *

Of Java + * and unsigned bytes. That is, its always a signed int in + * java no matter what the qualifier whether byte, char, etc. + * + *

Add accessors for optional filename, comment and MTIME. + * + * @author stack + */ +public class GzipHeader { + /** + * Length of minimal GZIP header. + * + * See RFC1952 for explaination of value of 10. + */ + public static final int MINIMAL_GZIP_HEADER_LENGTH = 10; + + /** + * Total length of the gzip header. + */ + protected int length = 0; + + /** + * The GZIP header FLG byte. + */ + protected int flg; + + /** + * GZIP header XFL byte. + */ + private int xfl; + + /** + * GZIP header OS byte. + */ + private int os; + + /** + * Extra header field content. + */ + private byte [] fextra = null; + + /** + * GZIP header MTIME field. + */ + private int mtime; + + + /** + * Shutdown constructor. + * + * Must pass an input stream. + */ + public GzipHeader() { + super(); + } + + /** + * Constructor. + * + * This constructor advances the stream past any gzip header found. + * + * @param in InputStream to read from. + * @throws IOException + */ + public GzipHeader(InputStream in) throws IOException { + super(); + readHeader(in); + } + + /** + * Read in gzip header. + * + * Advances the stream past the gzip header. + * @param in InputStream. + * + * @throws IOException Throws if does not start with GZIP Header. + */ + public void readHeader(InputStream in) throws IOException { + CRC32 crc = new CRC32(); + crc.reset(); + if (!testGzipMagic(in, crc)) { + throw new NoGzipMagicException(); + } + this.length += 2; + if (readByte(in, crc) != Deflater.DEFLATED) { + throw new IOException("Unknown compression"); + } + this.length++; + + // Get gzip header flag. + this.flg = readByte(in, crc); + this.length++; + + // Get MTIME. + this.mtime = readInt(in, crc); + this.length += 4; + + // Read XFL and OS. + this.xfl = readByte(in, crc); + this.length++; + this.os = readByte(in, crc); + this.length++; + + // Skip optional extra field -- stuff w/ alexa stuff in it. + final int FLG_FEXTRA = 4; + if ((this.flg & FLG_FEXTRA) == FLG_FEXTRA) { + int count = readShort(in, crc); + this.length +=2; + this.fextra = new byte[count]; + readByte(in, crc, this.fextra, 0, count); + this.length += count; + } + + // Skip file name. It ends in null. + final int FLG_FNAME = 8; + if ((this.flg & FLG_FNAME) == FLG_FNAME) { + while (readByte(in, crc) != 0) { + this.length++; + } + } + + // Skip file comment. It ends in null. + final int FLG_FCOMMENT = 16; // File comment + if ((this.flg & FLG_FCOMMENT) == FLG_FCOMMENT) { + while (readByte(in, crc) != 0) { + this.length++; + } + } + + // Check optional CRC. + final int FLG_FHCRC = 2; + if ((this.flg & FLG_FHCRC) == FLG_FHCRC) { + int calcCrc = (int)(crc.getValue() & 0xffff); + if (readShort(in, crc) != calcCrc) { + throw new IOException("Bad header CRC"); + } + this.length += 2; + } + } + + /** + * Test gzip magic is next in the stream. + * Reads two bytes. Caller needs to manage resetting stream. + * @param in InputStream to read. + * @return true if found gzip magic. False otherwise + * or an IOException (including EOFException). + * @throws IOException + */ + public boolean testGzipMagic(InputStream in) throws IOException { + return testGzipMagic(in, null); + } + + /** + * Test gzip magic is next in the stream. + * Reads two bytes. Caller needs to manage resetting stream. + * @param in InputStream to read. + * @param crc CRC to update. + * @return true if found gzip magic. False otherwise + * or an IOException (including EOFException). + * @throws IOException + */ + public boolean testGzipMagic(InputStream in, CRC32 crc) + throws IOException { + return readShort(in, crc) == GZIPInputStream.GZIP_MAGIC; + } + + /** + * Read an int. + * + * We do not expect to get a -1 reading. If we do, we throw exception. + * Update the crc as we go. + * + * @param in InputStream to read. + * @param crc CRC to update. + * @return int read. + * + * @throws IOException + */ + private int readInt(InputStream in, CRC32 crc) throws IOException { + int s = readShort(in, crc); + return ((readShort(in, crc) << 16) & 0xffff0000) | s; + } + + /** + * Read a short. + * + * We do not expect to get a -1 reading. If we do, we throw exception. + * Update the crc as we go. + * + * @param in InputStream to read. + * @param crc CRC to update. + * @return Short read. + * + * @throws IOException + */ + private int readShort(InputStream in, CRC32 crc) throws IOException { + int b = readByte(in, crc); + return ((readByte(in, crc) << 8) & 0x00ff00) | b; + } + + /** + * Read a byte. + * + * We do not expect to get a -1 reading. If we do, we throw exception. + * Update the crc as we go. + * + * @param in InputStream to read. + * @return Byte read. + * + * @throws IOException + */ + protected int readByte(InputStream in) throws IOException { + return readByte(in, null); + } + + /** + * Read a byte. + * + * We do not expect to get a -1 reading. If we do, we throw exception. + * Update the crc as we go. + * + * @param in InputStream to read. + * @param crc CRC to update. + * @return Byte read. + * + * @throws IOException + */ + protected int readByte(InputStream in, CRC32 crc) throws IOException { + int b = in.read(); + if (b == -1) { + throw new EOFException(); + } + if (crc != null) { + crc.update(b); + } + return b & 0xff; + } + + /** + * Read a byte. + * + * We do not expect to get a -1 reading. If we do, we throw exception. + * Update the crc as we go. + * + * @param in InputStream to read. + * @param crc CRC to update. + * @param buffer Buffer to read into. + * @param offset Offset to start filling buffer at. + * @param length How much to read. + * @return Bytes read. + * + * @throws IOException + */ + protected int readByte(InputStream in, CRC32 crc, byte [] buffer, + int offset, int length) + throws IOException { + for (int i = offset; i < length; i++) { + buffer[offset + i] = (byte)readByte(in, crc); + } + return length; + } + + /** + * @return Returns the fextra. + */ + public byte[] getFextra() { + return this.fextra; + } + + /** + * @return Returns the flg. + */ + public int getFlg() { + return this.flg; + } + + /** + * @return Returns the os. + */ + public int getOs() { + return this.os; + } + + /** + * @return Returns the xfl. + */ + public int getXfl() { + return this.xfl; + } + + /** + * @return Returns the mtime. + */ + public int getMtime() { + return this.mtime; + } + + /** + * @return Returns the length. + */ + public int getLength() { + return length; + } +} diff --git a/commons/src/main/java/org/archive/io/GzippedInputStream.java b/commons/src/main/java/org/archive/io/GzippedInputStream.java new file mode 100644 index 00000000..a1fc44ed --- /dev/null +++ b/commons/src/main/java/org/archive/io/GzippedInputStream.java @@ -0,0 +1,422 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io; + +import it.unimi.dsi.fastutil.io.RepositionableStream; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Iterator; +import java.util.logging.Logger; +import java.util.zip.Deflater; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; +import java.util.zip.Inflater; +import java.util.zip.ZipException; + + +/** + * Subclass of GZIPInputStream that can handle a stream made of multiple + * concatenated GZIP members/records. + * + * This class is needed because GZIPInputStream only finds the first GZIP + * member in the file even if the file is made up of multiple GZIP members. + * + *

Takes an InputStream stream that implements + * {@link RepositionableStream} interface so it can backup over-reads done + * by the zlib Inflater class. + * + *

Use the {@link #iterator()} method to get a gzip member iterator. + * Calls to {@link Iterator#next()} returns the next gzip member in the + * stream. Cast return from {@link Iterator#next()} to InputStream. + * + *

Use {@link #gzipMemberSeek(long)} to position stream before reading + * a gzip member if doing random accessing of gzip members. Pass it offset + * at which gzip member starts. + * + *

If you need to know position at which a gzip member starts, call + * {@link #position()} just after a call to {@link Iterator#hasNext()} + * and before you call {@link Iterator#next()}. + * + * @author stack + */ +public class GzippedInputStream +extends GZIPInputStream +implements RepositionableStream { + /** + * Tail on gzip members (The CRC). + */ + private static final int GZIP_TRAILER_LENGTH = 8; + + /** + * Utility class used probing for gzip members in stream. + * We need this instance to get at the readByte method. + */ + private final GzipHeader gzipHeader = new GzipHeader(); + + /** + * Buffer size used skipping over gzip members. + */ + private static final int LINUX_PAGE_SIZE = 4 * 1024; + + private final long initialOffset; + + public GzippedInputStream(InputStream is) throws IOException { + // Have buffer match linux page size. + this(is, LINUX_PAGE_SIZE); + } + + /** + * @param is An InputStream that implements RespositionableStream and + * returns true when we call + * {@link InputStream#markSupported()} (Latter is needed so can setup + * an {@link Iterator} against the Gzip stream). + * @param size Size of blocks to use reading. + * @throws IOException + */ + public GzippedInputStream(final InputStream is, final int size) + throws IOException { + super(checkStream(is), size); + if (!is.markSupported()) { + throw new IllegalArgumentException("GzippedInputStream requires " + + "a markable stream"); + } + if (!(is instanceof RepositionableStream)) { + throw new IllegalArgumentException("GzippedInputStream requires " + + "a stream that implements RepositionableStream"); + } + // We need to calculate the absolute offset of the current + // GZIP Member. Its almost always going to be zero but not + // always (We may have been passed a stream that is already part + // ways through a stream of GZIP Members). So, getting + // absolute offset is not exactly straight-forward. The super + // class, GZIPInputStream on construction reads in the GZIP Header + // which is a pain because I then do not know the absolute offset + // at which the GZIP record began. So, the call above to checkStream() + // marked the stream before passing it to the super calls. Then + // below we get current postion at just past the GZIP Header, call + // reset so we go back to the absolute start of the GZIP Member in + // the file, record the offset for later should we need to start + // over again in this file -- i.e. we're asked to get an iterator + // from Record zero on -- then we move the file position to just + // after the GZIP Header again so we're again aligned for inflation + // of the current record. + long afterGZIPHeader = ((RepositionableStream)is).position(); + is.reset(); + this.initialOffset = ((RepositionableStream)is).position(); + ((RepositionableStream)is).position(afterGZIPHeader); + } + + protected static InputStream checkStream(final InputStream is) + throws IOException { + if (is instanceof RepositionableStream) { + // See note above in constructor on why the mark here. + // Also minimal gzip header is 10. IA GZIP Headers are 20 bytes. + // Multiply by 4 in case extra info in the header. + is.mark(GzipHeader.MINIMAL_GZIP_HEADER_LENGTH * 4); + return is; + } + throw new IOException("Passed stream does not" + + " implement PositionableStream"); + } + + /** + * Exhaust current GZIP member content. + * Call this method when you think you're on the end of the + * GZIP member. It will clean out any dross. + * @param ignore Character to ignore counting characters (Usually + * trailing new lines). + * @return Count of characters skipped over. + * @throws IOException + */ + public long gotoEOR(int ignore) throws IOException { + long bytesSkipped = 0; + if (this.inf.getTotalIn() <= 0) { + return bytesSkipped; + } + if (!this.inf.finished()) { + int read = 0; + while ((read = read()) != -1) { + if ((byte)read == (byte)ignore) { + continue; + } + bytesSkipped = gotoEOR() + 1; + break; + } + } + return bytesSkipped; + } + + /** + * Exhaust current GZIP member content. + * Call this method when you think you're on the end of the + * GZIP member. It will clean out any dross. + * @return Count of characters skipped over. + * @throws IOException + */ + public long gotoEOR() throws IOException { + long bytesSkipped = 0; + if (this.inf.getTotalIn() <= 0) { + return bytesSkipped; + } + while(!this.inf.finished()) { + bytesSkipped += skip(Long.MAX_VALUE); + } + return bytesSkipped; + } + + /** + * Returns a GZIP Member Iterator. + * Has limitations. Can only get one Iterator per instance of this class; + * you must get new instance if you want to get Iterator again. + * @return Iterator over GZIP Members. + */ + public Iterator iterator() { + final Logger logger = Logger.getLogger(this.getClass().getName()); + + try { + // We know its a RepositionableStream else we'd have failed + // construction. On iterator construction, set file back to + // initial position so we're ready to read GZIP Members + // (May not always work dependent on how the + // RepositionableStream was implemented). + ((RepositionableStream)this.in).position(this.initialOffset); + } catch (IOException e) { + throw new RuntimeException(e); + } + return new Iterator() { + private GzippedInputStream compressedStream = + GzippedInputStream.this; + + public boolean hasNext() { + try { + gotoEOR(); + } catch (IOException e) { + if ((e instanceof ZipException) || + (e.getMessage() != null && + e.getMessage().startsWith("Corrupt GZIP trailer"))) { + // Try skipping end of bad record; try moving to next. + logger.info("Skipping exception " + e.getMessage()); + } else { + throw new RuntimeException(e); + } + } + return moveToNextGzipMember(); + } + + /** + * @return An InputStream onto a GZIP Member. + */ + public GzippedInputStream next() { + try { + gzipMemberSeek(); + } catch (IOException e) { + throw new RuntimeException("Failed move to EOR or " + + "failed header read: " + e.getMessage()); + } + return this.compressedStream; + } + + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } + + /** + * @return True if we found another record in the stream. + */ + protected boolean moveToNextGzipMember() { + boolean result = false; + // Move to the next gzip member, if there is one, positioning + // ourselves by backing up the stream so we reread any inflater + // remaining bytes. Then add 8 bytes to get us past the GZIP + // CRC trailer block that ends all gzip members. + try { + RepositionableStream ps = (RepositionableStream)getInputStream(); + // 8 is sizeof gzip CRC block thats on tail of gzipped + // record. If remaining is < 8 then experience indicates + // we're seeking past the gzip header -- don't backup the + // stream. + if (getInflater().getRemaining() > GZIP_TRAILER_LENGTH) { + ps.position(position() - getInflater().getRemaining() + + GZIP_TRAILER_LENGTH); + } + for (int read = -1, headerRead = 0; true; headerRead = 0) { + // Give a hint to underlying stream that we're going to want to + // do some backing up. + getInputStream().mark(3); + if ((read = getInputStream().read()) == -1) { + break; + } + if(compareBytes(read, GZIPInputStream.GZIP_MAGIC)) { + headerRead++; + if ((read = getInputStream().read()) == -1) { + break; + } + if(compareBytes(read, GZIPInputStream.GZIP_MAGIC >> 8)) { + headerRead++; + if ((read = getInputStream().read()) == -1) { + break; + } + if (compareBytes(read, Deflater.DEFLATED)) { + headerRead++; + // Found gzip header. Backup the stream the + // bytes we just found and set result true. + getInputStream().reset(); + result = true; + break; + } + } + // Didn't find gzip header. Reset stream but one byte + // futher on then redo header tests. + ps.position(ps.position() - headerRead); + } + } + } catch (IOException e) { + throw new RuntimeException("Failed i/o: " + e.getMessage()); + } + return result; + } + + protected boolean compareBytes(final int a, final int b) { + return ((byte)(a & 0xff)) == ((byte)(b & 0xff)); + } + + protected Inflater getInflater() { + return this.inf; + } + + protected InputStream getInputStream() { + return this.in; + } + + protected GzipHeader getGzipHeader() { + return this.gzipHeader; + } + + /** + * Move to next gzip member in the file. + */ + protected void resetInflater() { + this.eos = false; + this.inf.reset(); + } + + /** + * Read in the gzip header. + * @throws IOException + */ + protected void readHeader() throws IOException { + new GzipHeader(this.in); + // Reset the crc for subsequent reads. + this.crc.reset(); + } + + /** + * Seek to passed offset. + * + * After positioning the stream, it resets the inflater. + * Assumption is that public use of this method is only + * to position stream at start of a gzip member. + * + * @param position Absolute position of a gzip member start. + * @throws IOException + */ + public void position(long position) throws IOException { + ((RepositionableStream)this.in).position(position); + resetInflater(); + } + + public long position() throws IOException { + return ((RepositionableStream)this.in).position(); + } + + /** + * Seek to a gzip member. + * + * Moves stream to new position, resets inflater and reads in the gzip + * header ready for subsequent calls to read. + * + * @param position Absolute position of a gzip member start. + * @throws IOException + */ + public void gzipMemberSeek(long position) throws IOException { + position(position); + readHeader(); + } + + public void gzipMemberSeek() throws IOException { + gzipMemberSeek(position()); + } + + /** + * Gzip passed bytes. + * Use only when bytes is small. + * @param bytes What to gzip. + * @return A gzip member of bytes. + * @throws IOException + */ + public static byte [] gzip(byte [] bytes) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + GZIPOutputStream gzipOS = new GZIPOutputStream(baos); + gzipOS.write(bytes, 0, bytes.length); + gzipOS.close(); + return baos.toByteArray(); + } + + /** + * Tests passed stream is GZIP stream by reading in the HEAD. + * Does reposition of stream when done. + * @param rs An InputStream that is Repositionable. + * @return True if compressed stream. + * @throws IOException + */ + public static boolean isCompressedRepositionableStream( + final RepositionableStream rs) + throws IOException { + boolean result = false; + long p = rs.position(); + try { + result = isCompressedStream((InputStream)rs); + } finally { + rs.position(p); + } + return result; + } + + /** + * Tests passed stream is gzip stream by reading in the HEAD. + * Does not reposition stream when done. + * @param is An InputStream. + * @return True if compressed stream. + * @throws IOException + */ + public static boolean isCompressedStream(final InputStream is) + throws IOException { + try { + new GzipHeader(is); + } catch (NoGzipMagicException e) { + return false; + } + return true; + } +} diff --git a/commons/src/main/java/org/archive/io/HeaderedArchiveRecord.java b/commons/src/main/java/org/archive/io/HeaderedArchiveRecord.java new file mode 100644 index 00000000..2a669a84 --- /dev/null +++ b/commons/src/main/java/org/archive/io/HeaderedArchiveRecord.java @@ -0,0 +1,422 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintStream; + +import org.apache.commons.httpclient.Header; +import org.apache.commons.httpclient.HttpParser; +import org.apache.commons.httpclient.StatusLine; +import org.apache.commons.httpclient.util.EncodingUtil; +import org.archive.io.arc.ARCConstants; + +/** + * An ArchiveRecord whose content has a preamble of RFC822-like headers: e.g. + * The ArchiveRecord is a http response that leads off with http response + * headers. Use this ArchiveRecord Decorator to get at the content headers and + * the header/content demarcation. + * + * @author stack + * @author Olaf Freyer + */ +public class HeaderedArchiveRecord extends ArchiveRecord { + private int contentHeadersLength = -1; + private int statusCode = -1; + + /** + * Http header bytes. + * + * If non-null and bytes available, give out its contents before we + * go back to the underlying stream. + */ + private InputStream contentHeaderStream = null; + + /** + * Content headers. + * + * Only available after the reading of headers. + */ + private Header [] contentHeaders = null; + + + public HeaderedArchiveRecord(final ArchiveRecord ar) throws IOException { + super(ar); + } + + public HeaderedArchiveRecord(final ArchiveRecord ar, + final boolean readContentHeader) throws IOException { + super(ar); + if (readContentHeader) { + this.contentHeaderStream = readContentHeaders(); + } + } + + /** + * Skip over the the content headers if present. + * + * Subsequent reads will get the body. + * + *

Calling this method in the midst of reading the header + * will make for strange results. Otherwise, safe to call + * at any time though before reading any of the record + * content is only time that it makes sense. + * + *

After calling this method, you can call + * {@link #getContentHeaders()} to get the read http header. + * + * @throws IOException + */ + public void skipHttpHeader() throws IOException { + if (this.contentHeaderStream == null) { + return; + } + // Empty the contentHeaderStream + for (int available = this.contentHeaderStream.available(); + this.contentHeaderStream != null + && (available = this.contentHeaderStream.available()) > 0;) { + // We should be in this loop once only we should only do this + // buffer allocation once. + byte[] buffer = new byte[available]; + // The read nulls out httpHeaderStream when done with it so + // need check for null in the loop control line. + read(buffer, 0, available); + } + } + + public void dumpHttpHeader() throws IOException { + dumpHttpHeader(System.out); + } + + public void dumpHttpHeader(final PrintStream stream) throws IOException { + if (this.contentHeaderStream == null) { + return; + } + // Dump the httpHeaderStream to STDOUT + for (int available = this.contentHeaderStream.available(); + this.contentHeaderStream != null + && (available = this.contentHeaderStream.available()) > 0;) { + // We should be in this loop only once and should do this + // buffer allocation once. + byte[] buffer = new byte[available]; + // The read nulls out httpHeaderStream when done with it so + // need check for null in the loop control line. + int read = read(buffer, 0, available); + stream.write(buffer, 0, read); + } + } + + /** + * Read header if present. Technique borrowed from HttpClient HttpParse + * class. Using http parser code for now. Later move to more generic header + * parsing code if there proves a need. + * + * @return ByteArrayInputStream with the http header in it or null if no + * http header. + * @throws IOException + */ + private InputStream readContentHeaders() throws IOException { + // If judged a record that doesn't have an http header, return + // immediately. + if (!hasContentHeaders()) { + return null; + } + byte [] statusBytes = HttpParser.readRawLine(getIn()); + int eolCharCount = getEolCharsCount(statusBytes); + if (eolCharCount <= 0) { + throw new IOException("Failed to read raw lie where one " + + " was expected: " + new String(statusBytes)); + } + String statusLine = EncodingUtil.getString(statusBytes, 0, + statusBytes.length - eolCharCount, ARCConstants.DEFAULT_ENCODING); + if (statusLine == null) { + throw new NullPointerException("Expected status line is null"); + } + // TODO: Tighten up this test. + boolean isHttpResponse = StatusLine.startsWithHTTP(statusLine); + boolean isHttpRequest = false; + if (!isHttpResponse) { + isHttpRequest = statusLine.toUpperCase().startsWith("GET") || + !statusLine.toUpperCase().startsWith("POST"); + } + if (!isHttpResponse && !isHttpRequest) { + throw new UnexpectedStartLineIOException("Failed parse of " + + "status line: " + statusLine); + } + this.statusCode = isHttpResponse? + (new StatusLine(statusLine)).getStatusCode(): -1; + + // Save off all bytes read. Keep them as bytes rather than + // convert to strings so we don't have to worry about encodings + // though this should never be a problem doing http headers since + // its all supposed to be ascii. + ByteArrayOutputStream baos = + new ByteArrayOutputStream(statusBytes.length + 4 * 1024); + baos.write(statusBytes); + + // Now read rest of the header lines looking for the separation + // between header and body. + for (byte [] lineBytes = null; true;) { + lineBytes = HttpParser.readRawLine(getIn()); + eolCharCount = getEolCharsCount(lineBytes); + if (eolCharCount <= 0) { + throw new IOException("Failed reading headers: " + + ((lineBytes != null)? new String(lineBytes): null)); + } + // Save the bytes read. + baos.write(lineBytes); + if ((lineBytes.length - eolCharCount) <= 0) { + // We've finished reading the http header. + break; + } + } + + byte [] headerBytes = baos.toByteArray(); + // Save off where content body, post content headers, starts. + this.contentHeadersLength = headerBytes.length; + ByteArrayInputStream bais = + new ByteArrayInputStream(headerBytes); + if (!bais.markSupported()) { + throw new IOException("ByteArrayInputStream does not support mark"); + } + bais.mark(headerBytes.length); + // Read the status line. Don't let it into the parseHeaders function. + // It doesn't know what to do with it. + bais.read(statusBytes, 0, statusBytes.length); + this.contentHeaders = HttpParser.parseHeaders(bais, + ARCConstants.DEFAULT_ENCODING); + bais.reset(); + return bais; + } + + public static class UnexpectedStartLineIOException + extends RecoverableIOException { + private static final long serialVersionUID = 1L; + + public UnexpectedStartLineIOException(final String reason) { + super(reason); + } + } + + /** + * @param bytes Array of bytes to examine for an EOL. + * @return Count of end-of-line characters or zero if none. + */ + private int getEolCharsCount(byte [] bytes) { + int count = 0; + if (bytes != null && bytes.length >=1 && + bytes[bytes.length - 1] == '\n') { + count++; + if (bytes.length >=2 && bytes[bytes.length -2] == '\r') { + count++; + } + } + return count; + } + + /** + * @return If headers are for a http response AND the headers have been + * read, return status code. Else return -1. + */ + public int getStatusCode() { + return this.statusCode; + } + + /** + * @return Returns length of content headers or -1 if headers have + * not yet been read. + */ + public int getContentHeadersLength() { + return this.contentHeadersLength; + } + + public Header[] getContentHeaders() { + return contentHeaders; + } + + /** + * @return Next character in this ARCRecord's content else -1 if at end of + * this record. + * @throws IOException + */ + public int read() throws IOException { + int c = -1; + if (this.contentHeaderStream != null && + (this.contentHeaderStream.available() > 0)) { + // If http header, return bytes from it before we go to underlying + // stream. + c = this.contentHeaderStream.read(); + // If done with the header stream, null it out. + if (this.contentHeaderStream.available() <= 0) { + this.contentHeaderStream = null; + } + // do not increment position - + // the underlying ArchiveRecord stream allready did this + // incrementPosition(); + } else { + c = super.read(); + } + return c; + } + + public int read(byte [] b, int offset, int length) throws IOException { + int read = -1; + if (this.contentHeaderStream != null && + (this.contentHeaderStream.available() > 0)) { + // If http header, return bytes from it before we go to underlying + // stream. + read = Math.min(length, this.contentHeaderStream.available()); + if (read == 0) { + read = -1; + } else { + read = this.contentHeaderStream.read(b, offset, read); + } + // If done with the header stream, null it out. + if (this.contentHeaderStream.available() <= 0) { + this.contentHeaderStream = null; + } + // do not increment position - + // the underlying ArchiveRecord stream allready did this + //incrementPosition(); + } else { + read = super.read(b, offset, length); + } + return read; + } + + @Override + public int available() { + return ((ArchiveRecord)this.in).available(); + } + + @Override + public void close() throws IOException { + ((ArchiveRecord)this.in).close(); + } + + @Override + public void dump() throws IOException { + ((ArchiveRecord)this.in).dump(); + } + + @Override + public void dump(OutputStream os) throws IOException { + ((ArchiveRecord)this.in).dump(os); + } + + @Override + protected String getDigest4Cdx(ArchiveRecordHeader h) { + return ((ArchiveRecord)this.in).getDigest4Cdx(h); + } + + @Override + public String getDigestStr() { + return ((ArchiveRecord)this.in).getDigestStr(); + } + + @Override + public ArchiveRecordHeader getHeader() { + return ((ArchiveRecord)this.in).getHeader(); + } + + @Override + protected String getIp4Cdx(ArchiveRecordHeader h) { + return ((ArchiveRecord)this.in).getIp4Cdx(h); + } + + @Override + protected String getMimetype4Cdx(ArchiveRecordHeader h) { + return ((ArchiveRecord)this.in).getMimetype4Cdx(h); + } + + @Override + protected long getPosition() { + return ((ArchiveRecord)this.in).getPosition(); + } + + @Override + protected String getStatusCode4Cdx(ArchiveRecordHeader h) { + return ((ArchiveRecord)this.in).getStatusCode4Cdx(h); + } + + @Override + public boolean hasContentHeaders() { + return ((ArchiveRecord)this.in).hasContentHeaders(); + } + + @Override + protected void incrementPosition() { + ((ArchiveRecord)this.in).incrementPosition(); + } + + @Override + protected void incrementPosition(long incr) { + ((ArchiveRecord)this.in).incrementPosition(incr); + } + + @Override + protected boolean isEor() { + return ((ArchiveRecord)this.in).isEor(); + } + + @Override + public boolean isStrict() { + return ((ArchiveRecord)this.in).isStrict(); + } + + @Override + public boolean markSupported() { + return ((ArchiveRecord)this.in).markSupported(); + } + + @Override + protected String outputCdx(String strippedFileName) throws IOException { + return ((ArchiveRecord)this.in).outputCdx(strippedFileName); + } + + @Override + protected void setEor(boolean eor) { + ((ArchiveRecord)this.in).setEor(eor); + } + + @Override + protected void setHeader(ArchiveRecordHeader header) { + ((ArchiveRecord)this.in).setHeader(header); + } + + @Override + public void setStrict(boolean strict) { + ((ArchiveRecord)this.in).setStrict(strict); + } + + @Override + void skip() throws IOException { + ((ArchiveRecord)this.in).skip(); + } + + @Override + public long skip(long n) throws IOException { + return ((ArchiveRecord)this.in).skip(n); + } +} diff --git a/commons/src/main/java/org/archive/io/LoudObjectOutputStream.java b/commons/src/main/java/org/archive/io/LoudObjectOutputStream.java new file mode 100644 index 00000000..9d6c52ef --- /dev/null +++ b/commons/src/main/java/org/archive/io/LoudObjectOutputStream.java @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2007 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * LoudObjectOutputStream.java + * + * Created on Mar 10, 2007 + * + * $Id:$ + */ + +package org.archive.io; + +import java.io.IOException; +import java.io.ObjectOutputStream; +import java.io.OutputStream; +import java.util.HashSet; +import java.util.Set; +import java.util.logging.Logger; + +/** + * ObjectOutputStream that logs class name of each object that is written + * to the stream. Useful for tracking down sources of NotSerializableException. + * + * @author pjack + * + */ +public class LoudObjectOutputStream extends ObjectOutputStream { + + + final private static Logger LOGGER = Logger.getLogger( + LoudObjectOutputStream.class.getName()); + + // Only log each class name once + private Set alreadyLogged = new HashSet(); + + public LoudObjectOutputStream(OutputStream out) throws IOException { + super(out); + this.enableReplaceObject(true); + } + + + @Override + protected Object replaceObject(Object obj) throws IOException { + if (obj != null) { + String name = obj.getClass().getName(); + if (alreadyLogged.add(name)) { + LOGGER.info("WROTE: " + name); + } + } + return obj; + } + + +} diff --git a/commons/src/main/java/org/archive/io/MultiByteReplayCharSequence.java b/commons/src/main/java/org/archive/io/MultiByteReplayCharSequence.java new file mode 100644 index 00000000..187ef917 --- /dev/null +++ b/commons/src/main/java/org/archive/io/MultiByteReplayCharSequence.java @@ -0,0 +1,346 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.Charset; +import java.util.logging.Logger; + +/** + * Provides a (Replay)CharSequence view on recorded streams (a prefix + * buffer and overflow backing file) that can handle streams of multibyte + * characters. + * + * If possible, use {@link ByteReplayCharSequence}. It performs better even + * for the single byte case (Decoding is an expensive process). + * + *

Call close on this class when done so can clean up resources. + * + *

Implementation currently works by checking to see if content to read + * all fits the in-memory buffer. If so, we decode into a CharBuffer and + * keep this around for CharSequence operations. This CharBuffer is + * discarded on close. + * + *

If content length is greater than in-memory buffer, we decode the + * buffer plus backing file into a new file named for the backing file w/ + * a suffix of the encoding we write the file as. We then run w/ a + * memory-mapped CharBuffer against this file to implement CharSequence. + * Reasons for this implemenation are that CharSequence wants to return the + * length of the CharSequence. + * + *

Obvious optimizations would keep around decodings whether the + * in-memory decoded buffer or the file of decodings written to disk but the + * general usage pattern processing URIs is that the decoding is used by one + * processor only. Also of note, files usually fit into the in-memory + * buffer. + * + *

We might also be able to keep up 3 windows that moved across the file + * decoding a window at a time trying to keep one of the buffers just in + * front of the regex processing returning it a length that would be only + * the length of current position to end of current block or else the length + * could be got by multipling the backing files length by the decoders' + * estimate of average character size. This would save us writing out the + * decoded file. We'd have to do the latter for files that are + * > Integer.MAX_VALUE. + * + * @author stack + * @version $Revision$, $Date$ + */ +public class MultiByteReplayCharSequence implements ReplayCharSequence { + + protected static Logger logger = + Logger.getLogger(MultiByteReplayCharSequence.class.getName()); + + /** + * Name of the encoding we use writing out concatenated decoded prefix + * buffer and decoded backing file. + * + *

This define is also used as suffix for the file that holds the + * decodings. The name of the file that holds the decoding is the name + * of the backing file w/ this encoding for a suffix. + * + *

See Encoding. + */ + private static final String WRITE_ENCODING = "UTF-16BE"; + + /** + * CharBuffer of decoded content. + * + * Content of this buffer is unicode. + */ + private CharBuffer content = null; + + /** + * File that has decoded content. + * + * Keep it around so we can remove on close. + */ + private File decodedFile = null; + + + /** + * Constructor for all in-memory operation. + * + * @param buffer In-memory buffer of recordings prefix. We read from + * here first and will only go to the backing file if size + * requested is greater than buffer.length. + * @param size Total size of stream to replay in bytes. Used to find + * EOS. This is total length of content including HTTP headers if + * present. + * @param responseBodyStart Where the response body starts in bytes. + * Used to skip over the HTTP headers if present. + * @param backingFilename Path to backing file with content in excess of + * whats in buffer. + * @param encoding Encoding to use reading the passed prefix buffer and + * backing file. For now, should be java canonical name for the + * encoding. (If null is passed, we will default to + * ByteReplayCharSequence). + * + * @throws IOException + */ + public MultiByteReplayCharSequence(byte[] buffer, long size, + long responseBodyStart, String encoding) + throws IOException { + super(); + this.content = decodeInMemory(buffer, size, responseBodyStart, + encoding); + } + + /** + * Constructor for overflow-to-disk-file operation. + * + * @param contentReplayInputStream inputStream of content + * @param backingFilename hint for name of temp file + * @param characterEncoding Encoding to use reading the stream. + * For now, should be java canonical name for the + * encoding. + * + * @throws IOException + */ + public MultiByteReplayCharSequence( + ReplayInputStream contentReplayInputStream, + String backingFilename, + String characterEncoding) + throws IOException { + super(); + this.content = decodeToFile(contentReplayInputStream, + backingFilename, characterEncoding); + } + + /** + * Decode passed buffer and backing file into a CharBuffer. + * + * This method writes a new file made of the decoded concatenation of + * the in-memory prefix buffer and the backing file. Returns a + * charSequence view onto this new file. + * + * @param buffer In-memory buffer of recordings prefix. We read from + * here first and will only go to the backing file if size + * requested is greater than buffer.length. + * @param size Total size of stream to replay in bytes. Used to find + * EOS. This is total length of content including HTTP headers if + * present. + * @param responseBodyStart Where the response body starts in bytes. + * Used to skip over the HTTP headers if present. + * @param backingFilename Path to backing file with content in excess of + * whats in buffer. + * @param encoding Encoding to use reading the passed prefix buffer and + * backing file. For now, should be java canonical name for the + * encoding. (If null is passed, we will default to + * ByteReplayCharSequence). + * + * @return A CharBuffer view on decodings of the contents of passed + * buffer. + * @throws IOException + */ + private CharBuffer decodeToFile(ReplayInputStream inStream, + String backingFilename, String encoding) + throws IOException { + + CharBuffer charBuffer = null; + + BufferedReader reader = new BufferedReader( + new InputStreamReader(inStream,encoding)); + + this.decodedFile = new File(backingFilename + "." + WRITE_ENCODING); + FileOutputStream fos; + try { + fos = new FileOutputStream(this.decodedFile); + } catch (FileNotFoundException e) { + // Windows workaround attempt + System.gc(); + System.runFinalization(); + logger.info("Windows 'file with a user-mapped section open' "+ + "workaround gc-finalization performed."); + // try again + fos = new FileOutputStream(this.decodedFile); + } + BufferedWriter writer = new BufferedWriter( + new OutputStreamWriter( + fos, + WRITE_ENCODING)); + + int c; + while((c = reader.read())>=0) { + writer.write(c); + } + writer.close(); + + charBuffer = getReadOnlyMemoryMappedBuffer(this.decodedFile). + asCharBuffer(); + + return charBuffer; + } + + /** + * Decode passed buffer into a CharBuffer. + * + * This method decodes a memory buffer returning a memory buffer. + * + * @param buffer In-memory buffer of recordings prefix. We read from + * here first and will only go to the backing file if size + * requested is greater than buffer.length. + * @param size Total size of stream to replay in bytes. Used to find + * EOS. This is total length of content including HTTP headers if + * present. + * @param responseBodyStart Where the response body starts in bytes. + * Used to skip over the HTTP headers if present. + * @param encoding Encoding to use reading the passed prefix buffer and + * backing file. For now, should be java canonical name for the + * encoding. (If null is passed, we will default to + * ByteReplayCharSequence). + * + * @return A CharBuffer view on decodings of the contents of passed + * buffer. + */ + private CharBuffer decodeInMemory(byte[] buffer, long size, + long responseBodyStart, String encoding) + { + ByteBuffer bb = ByteBuffer.wrap(buffer); + // Move past the HTTP header if present. + bb.position((int)responseBodyStart); + // Set the end-of-buffer to be end-of-content. + bb.limit((int)size); + return (Charset.forName(encoding)).decode(bb).asReadOnlyBuffer(); + } + + /** + * Create read-only memory-mapped buffer onto passed file. + * + * @param file File to get memory-mapped buffer on. + * @return Read-only memory-mapped ByteBuffer view on to passed file. + * @throws IOException + */ + private ByteBuffer getReadOnlyMemoryMappedBuffer(File file) + throws IOException { + + ByteBuffer bb = null; + FileInputStream in = null; + FileChannel c = null; + assert file.exists(): "No file " + file.getAbsolutePath(); + + try { + in = new FileInputStream(file); + c = in.getChannel(); + // TODO: Confirm the READ_ONLY works. I recall it not working. + // The buffers seem to always say that the buffer is writeable. + bb = c.map(FileChannel.MapMode.READ_ONLY, 0, c.size()). + asReadOnlyBuffer(); + } + + finally { + if (c != null && c.isOpen()) { + c.close(); + } + if (in != null) { + in.close(); + } + } + + return bb; + } + + private void deleteFile(File fileToDelete) { + deleteFile(fileToDelete, null); + } + + private void deleteFile(File fileToDelete, final Exception e) { + if (e != null) { + // Log why the delete to help with debug of java.io.FileNotFoundException: + // ....tt53http.ris.UTF-16BE. + logger.severe("Deleting " + fileToDelete + " because of " + + e.toString()); + } + if (fileToDelete != null && fileToDelete.exists()) { + fileToDelete.delete(); + } + } + + public void close() + { + this.content = null; + deleteFile(this.decodedFile); + // clear decodedFile -- so that double-close (as in + // finalize()) won't delete a later instance with same name + // see bug [ 1218961 ] "failed get of replay" in ExtractorHTML... usu: UTF-16BE + this.decodedFile = null; + } + + protected void finalize() throws Throwable + { + super.finalize(); + // Maybe TODO: eliminate close here, requiring explicit close instead + close(); + } + + public int length() + { + return this.content.limit(); + } + + public char charAt(int index) + { + return this.content.get(index); + } + + public CharSequence subSequence(int start, int end) { + return new CharSubSequence(this, start, end); + } + + public String toString() { + StringBuffer sb = new StringBuffer(length()); + // could use StringBuffer.append(CharSequence) if willing to do 1.5 & up + for (int i = 0;i auxiliaryDirectoryStack = new LinkedList(); + LinkedList postRestoreTasks = new LinkedList(); + + /** + * Instantiate over the given stream and using the supplied + * auxiliary storage directory. + * + * @param in + * @param storeDir + * @throws IOException + */ + public ObjectPlusFilesInputStream(InputStream in, File storeDir) + throws IOException { + super(in); + auxiliaryDirectoryStack.addFirst(storeDir); + } + + /** + * Push another default storage directory for use + * until popped. + * + * @param dir + */ + public void pushAuxiliaryDirectory(String dir) { + auxiliaryDirectoryStack. + addFirst(new File(getAuxiliaryDirectory(), dir)); + } + + /** + * Discard the top auxiliary directory. + */ + public void popAuxiliaryDirectory() { + auxiliaryDirectoryStack.removeFirst(); + } + + /** + * Return the top auxiliary directory, from + * which saved files are restored. + * + * @return Auxillary directory. + */ + public File getAuxiliaryDirectory() { + return (File)auxiliaryDirectoryStack.getFirst(); + } + + /** + * Restore a file from storage, using the name and length + * info on the serialization stream and the file from the + * current auxiliary directory, to the given File. + * + * @param destination + * @throws IOException + */ + public void restoreFile(File destination) throws IOException { + String nameAsStored = readUTF(); + long lengthAtStoreTime = readLong(); + File storedFile = new File(getAuxiliaryDirectory(),nameAsStored); + FileUtils.copyFile(storedFile, destination, lengthAtStoreTime); + } + + /** + * Restore a file from storage, using the name and length + * info on the serialization stream and the file from the + * current auxiliary directory, to the given File. + * + * @param directory + * @throws IOException + */ + public void restoreFileTo(File directory) throws IOException { + String nameAsStored = readUTF(); + long lengthAtStoreTime = readLong(); + File storedFile = new File(getAuxiliaryDirectory(),nameAsStored); + File destination = new File(directory,nameAsStored); + FileUtils.copyFile(storedFile, destination, lengthAtStoreTime); + } + + /** + * Register a task to be done when the ObjectPlusFilesInputStream + * is closed. + * + * @param task + */ + public void registerFinishTask(Runnable task) { + postRestoreTasks.addFirst(task); + } + + private void doFinishTasks() { + Iterator iter = postRestoreTasks.iterator(); + while(iter.hasNext()) { + ((Runnable)iter.next()).run(); + } + } + + /** + * In addition to default, do any registered cleanup tasks. + * + * @see java.io.InputStream#close() + */ + public void close() throws IOException { + super.close(); + doFinishTasks(); + } +} diff --git a/commons/src/main/java/org/archive/io/ObjectPlusFilesOutputStream.java b/commons/src/main/java/org/archive/io/ObjectPlusFilesOutputStream.java new file mode 100644 index 00000000..7e500c3a --- /dev/null +++ b/commons/src/main/java/org/archive/io/ObjectPlusFilesOutputStream.java @@ -0,0 +1,140 @@ +/* ObjectPlusFilesOutputStream +* +* $Id$ +* +* Created on Apr 28, 2004 +* +* Copyright (C) 2004 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.io; + +import java.io.File; +import java.io.IOException; +import java.io.ObjectOutputStream; +import java.io.OutputStream; +import java.util.LinkedList; + +import org.archive.util.FileUtils; + + +/** + * Enhanced ObjectOutputStream which maintains (a stack of) auxiliary + * directories and offers convenience methods for serialized objects + * to save their related disk files alongside their serialized version. + * + * @author gojomo + */ +public class ObjectPlusFilesOutputStream extends ObjectOutputStream { + LinkedList auxiliaryDirectoryStack = new LinkedList(); + + /** + * Constructor + * + * @param out + * @param topDirectory + * @throws java.io.IOException + */ + public ObjectPlusFilesOutputStream(OutputStream out, File topDirectory) throws IOException { + super(out); + auxiliaryDirectoryStack.addFirst(topDirectory); + } + + /** + * Add another subdirectory for any file-capture needs during the + * current serialization. + * + * @param dir + */ + public void pushAuxiliaryDirectory(String dir) { + auxiliaryDirectoryStack.addFirst(new File(getAuxiliaryDirectory(),dir)); + } + + /** + * Remove the top subdirectory. + * + */ + public void popAuxiliaryDirectory() { + auxiliaryDirectoryStack.removeFirst(); + } + + /** + * Return the current auxiliary directory for storing + * files associated with serialized objects. + * + * @return Auxillary directory. + */ + public File getAuxiliaryDirectory() { + return (File)auxiliaryDirectoryStack.getFirst(); + } + + /** + * Store a snapshot of an object's supporting file to the + * current auxiliary directory. Should only be used for + * files which are strictly appended-to, because it tries + * to use a "hard link" where possible (meaning that + * future edits to the original file's contents will + * also affect the snapshot). + * + * Remembers current file extent to allow a future restore + * to ignore subsequent appended data. + * + * @param file + * @throws IOException + */ + public void snapshotAppendOnlyFile(File file) throws IOException { + // write filename + String name = file.getName(); + writeUTF(name); + // write current file length + writeLong(file.length()); + File auxDir = getAuxiliaryDirectory(); + if(!auxDir.exists()) { + auxDir.mkdirs(); + } + File destination = new File(auxDir,name); + hardlinkOrCopy(file, destination); + } + + /** + * Create a backup of this given file, first by trying a "hard + * link", then by using a copy if hard linking is unavailable + * (either because it is unsupported or the origin and checkpoint + * directories are on different volumes). + * + * @param file + * @param destination + * @throws IOException + */ + private void hardlinkOrCopy(File file, File destination) throws IOException { + // For Linux/UNIX, try a hard link first. + Process link = Runtime.getRuntime().exec("ln "+file.getAbsolutePath()+" "+destination.getAbsolutePath()); + // TODO NTFS also supports hard links; add appropriate try + try { + link.waitFor(); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + if(link.exitValue()!=0) { + // hard link failed + FileUtils.copyFile(file,destination); + } + } + +} diff --git a/commons/src/main/java/org/archive/io/OriginSeekInputStream.java b/commons/src/main/java/org/archive/io/OriginSeekInputStream.java new file mode 100644 index 00000000..b13d7fc6 --- /dev/null +++ b/commons/src/main/java/org/archive/io/OriginSeekInputStream.java @@ -0,0 +1,125 @@ +/* OriginSeekInputStream +* +* Created on September 14, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.io; + + +import java.io.IOException; + + +/** + * Alters the origin of some other SeekInputStream. This class allows you + * to completely ignore everything in the underlying stream before a specified + * position, the origin position. + * + *

With the exception of {@link #position()} and {@link position(long)}, + * all of the methods in this class simply delegate to the underlying input + * stream. The position methods adjust the position of the + * underlying stream relative to the origin specified at construction time. + * + * @author pjack + */ +public class OriginSeekInputStream extends SeekInputStream { + + + /** + * The underlying stream. + */ + final private SeekInputStream input; + + + /** + * The origin position. In other words, this.position(0) + * resolves to input.position(start). + */ + final private long origin; + + + /** + * Constructor. + * + * @param input the underlying stream + * @param origin the origin position + * @throws IOException if an IO error occurs + */ + public OriginSeekInputStream(SeekInputStream input, long origin) + throws IOException { + this.input = input; + this.origin = origin; + input.position(origin); + } + + + @Override + public int available() throws IOException { + return input.available(); + } + + + @Override + public int read() throws IOException { + return input.read(); + } + + + @Override + public int read(byte[] buf, int ofs, int len) throws IOException { + return input.read(buf, ofs, len); + } + + + @Override + public int read(byte[] buf) throws IOException { + return input.read(buf); + } + + + @Override + public long skip(long count) throws IOException { + return input.skip(count); + } + + + /** + * Returns the position of the underlying stream relative to the origin. + * + * @return the relative position + * @throws IOException if an IO error occurs + */ + public long position() throws IOException { + return input.position() - origin; + } + + + /** + * Positions the underlying stream relative to the origin. + * In other words, this.position(0) resolves to input.position(origin), + * where input is underlying stream and origin is the origin specified + * at construction time. + * + * @param p the new position for this stream + * @throws IOException if an IO error occurs + */ + public void position(long p) throws IOException { + input.position(p + origin); + } +} diff --git a/commons/src/main/java/org/archive/io/RandomAccessInputStream.java b/commons/src/main/java/org/archive/io/RandomAccessInputStream.java new file mode 100644 index 00000000..9b6bee77 --- /dev/null +++ b/commons/src/main/java/org/archive/io/RandomAccessInputStream.java @@ -0,0 +1,184 @@ +/* RandomAccessInputStream +* +* Created on May 21, 2004 +* +* Copyright (C) 2004 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.io; + + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; + + +/** + * Wraps a RandomAccessFile with an InputStream interface. + * + * @author gojomo + */ +public class RandomAccessInputStream extends SeekInputStream { + + /** + * Reference to the random access file this stream is reading from. + */ + private RandomAccessFile raf = null; + + /** + * When mark is called, save here the current position so we can go back + * on reset. + */ + private long markpos = -1; + + /** + * True if we are to close the underlying random access file when this + * stream is closed. + */ + private boolean sympathyClose; + + /** + * Constructor. + * + * If using this constructor, caller created the RAF and therefore + * its assumed wants to control close of the RAF. The RAF.close + * is not called if this constructor is used on close of this stream. + * + * @param raf RandomAccessFile to wrap. + * @throws IOException + */ + public RandomAccessInputStream(RandomAccessFile raf) + throws IOException { + this(raf, false, 0); + } + + /** + * Constructor. + * + * @param file File to get RAFIS on. Creates an RAF from passed file. + * Closes the created RAF when this stream is closed. + * @throws IOException + */ + public RandomAccessInputStream(final File file) + throws IOException { + this(new RandomAccessFile(file, "r"), true, 0); + } + + /** + * Constructor. + * + * @param file File to get RAFIS on. Creates an RAF from passed file. + * Closes the created RAF when this stream is closed. + * @param offset + * @throws IOException + */ + public RandomAccessInputStream(final File file, final long offset) + throws IOException { + this(new RandomAccessFile(file, "r"), true, offset); + } + + /** + * @param raf RandomAccessFile to wrap. + * @param sympathyClose Set to true if we are to close the RAF + * file when this stream is closed. + * @param offset + * @throws IOException + */ + public RandomAccessInputStream(final RandomAccessFile raf, + final boolean sympathyClose, final long offset) + throws IOException { + super(); + this.sympathyClose = sympathyClose; + this.raf = raf; + if (offset > 0) { + this.raf.seek(offset); + } + } + + /* (non-Javadoc) + * @see java.io.InputStream#read() + */ + public int read() throws IOException { + return this.raf.read(); + } + + /* (non-Javadoc) + * @see java.io.InputStream#read(byte[], int, int) + */ + public int read(byte[] b, int off, int len) throws IOException { + return this.raf.read(b, off, len); + } + + /* (non-Javadoc) + * @see java.io.InputStream#read(byte[]) + */ + public int read(byte[] b) throws IOException { + return this.raf.read(b); + } + + /* (non-Javadoc) + * @see java.io.InputStream#skip(long) + */ + public long skip(long n) throws IOException { + this.raf.seek(this.raf.getFilePointer() + n); + return n; + } + + public long position() throws IOException { + return this.raf.getFilePointer(); + } + + public void position(long position) throws IOException { + this.raf.seek(position); + } + + public int available() throws IOException { + long amount = this.raf.length() - this.position(); + return (amount >= Integer.MAX_VALUE)? Integer.MAX_VALUE: (int)amount; + } + + public boolean markSupported() { + return true; + } + + public synchronized void mark(int readlimit) { + try { + this.markpos = position(); + } catch (IOException e) { + // Set markpos to -1. Will cause exception reset. + this.markpos = -1; + } + } + + public synchronized void reset() throws IOException { + if (this.markpos == -1) { + throw new IOException("Mark has not been set."); + } + position(this.markpos); + } + + public void close() throws IOException { + try { + super.close(); + } finally { + if (this.sympathyClose) { + this.raf.close(); + } + } + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/io/RandomAccessOutputStream.java b/commons/src/main/java/org/archive/io/RandomAccessOutputStream.java new file mode 100644 index 00000000..a1caec90 --- /dev/null +++ b/commons/src/main/java/org/archive/io/RandomAccessOutputStream.java @@ -0,0 +1,74 @@ +/* Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * RandomAccessOutputStream.java + * Created on May 21, 2004 + * + * $Header$ + */ +package org.archive.io; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.RandomAccessFile; + + +/** + * Wraps a RandomAccessFile with OutputStream interface. + * + * @author gojomo + */ +public class RandomAccessOutputStream extends OutputStream { + RandomAccessFile raf; + + /** + * Wrap the given RandomAccessFile + */ + public RandomAccessOutputStream(RandomAccessFile raf) { + super(); + this.raf = raf; + } + + /* (non-Javadoc) + * @see java.io.OutputStream#write(int) + */ + public void write(int b) throws IOException { + raf.write(b); + } + + /* (non-Javadoc) + * @see java.io.OutputStream#close() + */ + public void close() throws IOException { + raf.close(); + } + + /* (non-Javadoc) + * @see java.io.OutputStream#write(byte[], int, int) + */ + public void write(byte[] b, int off, int len) throws IOException { + raf.write(b, off, len); + } + + /* (non-Javadoc) + * @see java.io.OutputStream#write(byte[]) + */ + public void write(byte[] b) throws IOException { + raf.write(b); + } +} diff --git a/commons/src/main/java/org/archive/io/ReadSource.java b/commons/src/main/java/org/archive/io/ReadSource.java new file mode 100644 index 00000000..9721bc1e --- /dev/null +++ b/commons/src/main/java/org/archive/io/ReadSource.java @@ -0,0 +1,26 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io; + +import java.io.Reader; + +public interface ReadSource { + Reader getReader(); +} diff --git a/commons/src/main/java/org/archive/io/RecorderIOException.java b/commons/src/main/java/org/archive/io/RecorderIOException.java new file mode 100644 index 00000000..42c8e076 --- /dev/null +++ b/commons/src/main/java/org/archive/io/RecorderIOException.java @@ -0,0 +1,43 @@ +/* Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * RecorderIOException.java + * Created on Oct 22, 2003 + * + * $Header$ + */ +package org.archive.io; + +import java.io.IOException; + +/** + * + * @author Gordon Mohr + */ +public class RecorderIOException extends IOException { + + private static final long serialVersionUID = 5907470275350314277L; + + public RecorderIOException() { + super(); + } + + public RecorderIOException(String msg) { + super(msg); + } +} diff --git a/commons/src/main/java/org/archive/io/RecorderLengthExceededException.java b/commons/src/main/java/org/archive/io/RecorderLengthExceededException.java new file mode 100644 index 00000000..c4a5eda1 --- /dev/null +++ b/commons/src/main/java/org/archive/io/RecorderLengthExceededException.java @@ -0,0 +1,44 @@ +/* Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * RecorderLengthExceededException.java + * Created on Oct 22, 2003 + * + * $Header$ + */ +package org.archive.io; + + +/** + * Indicates a length exception thrown by the Recorder. + * + * @author Gordon Mohr + */ +public class RecorderLengthExceededException +extends RecorderIOException { + + private static final long serialVersionUID = 6655419033414648444L; + + public RecorderLengthExceededException() { + super(); + } + + public RecorderLengthExceededException(String msg) { + super(msg); + } +} diff --git a/commons/src/main/java/org/archive/io/RecorderTimeoutException.java b/commons/src/main/java/org/archive/io/RecorderTimeoutException.java new file mode 100644 index 00000000..6efa875a --- /dev/null +++ b/commons/src/main/java/org/archive/io/RecorderTimeoutException.java @@ -0,0 +1,42 @@ +/* Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * RecorderTimeoutException.java + * Created on Oct 22, 2003 + * + * $Header$ + */ +package org.archive.io; + +/** + * Indicates a timeout thrown by the RecordingInputStream. + * + * @author Gordon Mohr + */ +public class RecorderTimeoutException extends RecorderIOException { + + private static final long serialVersionUID = 7433214063765078269L; + + public RecorderTimeoutException() { + super(); + } + + public RecorderTimeoutException(String msg) { + super(msg); + } +} diff --git a/commons/src/main/java/org/archive/io/RecorderTooMuchHeaderException.java b/commons/src/main/java/org/archive/io/RecorderTooMuchHeaderException.java new file mode 100644 index 00000000..f2dc2ab7 --- /dev/null +++ b/commons/src/main/java/org/archive/io/RecorderTooMuchHeaderException.java @@ -0,0 +1,45 @@ +/* Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * RecorderLengthExceededException.java + * Created on Oct 22, 2003 + * + * $Header$ + */ +package org.archive.io; + + +/** + * Indicates a too much header material exception thrown by the Recorder + * (specificially the RecordingOutputStream) + * + * @author Gordon Mohr + */ +public class RecorderTooMuchHeaderException +extends RecorderIOException { + + private static final long serialVersionUID = 3528516034898129150L; + + public RecorderTooMuchHeaderException() { + super(); + } + + public RecorderTooMuchHeaderException(String msg) { + super(msg); + } +} diff --git a/commons/src/main/java/org/archive/io/RecordingInputStream.java b/commons/src/main/java/org/archive/io/RecordingInputStream.java new file mode 100644 index 00000000..3a00ce75 --- /dev/null +++ b/commons/src/main/java/org/archive/io/RecordingInputStream.java @@ -0,0 +1,369 @@ +/* RecordingInputStream + * + * $Id$ + * + * Created on Sep 24, 2003 + * + * Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.io; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.SocketException; +import java.net.SocketTimeoutException; +import java.security.MessageDigest; +import java.util.logging.Level; +import java.util.logging.Logger; + + +/** + * Stream which records all data read from it, which it acquires from a wrapped + * input stream. + * + * Makes use of a RecordingOutputStream for recording because of its being + * file backed so we can write massive amounts of data w/o worrying about + * overflowing memory. + * + * @author gojomo + * + */ +public class RecordingInputStream + extends InputStream { + + protected static Logger logger = + Logger.getLogger("org.archive.io.RecordingInputStream"); + + /** + * Where we are recording to. + */ + private RecordingOutputStream recordingOutputStream; + + /** + * Stream to record. + */ + private InputStream in = null; + + /** + * Reusable buffer to avoid reallocation on each readFullyUntil + */ + protected byte[] drainBuffer = new byte[16*1024]; + + /** + * Create a new RecordingInputStream. + * + * @param bufferSize Size of buffer to use. + * @param backingFilename Name of backing file. + */ + public RecordingInputStream(int bufferSize, String backingFilename) + { + this.recordingOutputStream = new RecordingOutputStream(bufferSize, + backingFilename); + } + + public void open(InputStream wrappedStream) throws IOException { + logger.fine(Thread.currentThread().getName() + " opening " + + wrappedStream + ", " + Thread.currentThread().getName()); + if(isOpen()) { + // error; should not be opening/wrapping in an unclosed + // stream remains open + throw new IOException("RIS already open for " + +Thread.currentThread().getName()); + } + this.in = wrappedStream; + this.recordingOutputStream.open(); + } + + public int read() throws IOException { + if (!isOpen()) { + throw new IOException("Stream closed " + + Thread.currentThread().getName()); + } + int b = this.in.read(); + if (b != -1) { + assert this.recordingOutputStream != null: "ROS is null " + + Thread.currentThread().getName(); + this.recordingOutputStream.write(b); + } + return b; + } + + public int read(byte[] b, int off, int len) throws IOException { + if (!isOpen()) { + throw new IOException("Stream closed " + + Thread.currentThread().getName()); + } + int count = this.in.read(b,off,len); + if (count > 0) { + assert this.recordingOutputStream != null: "ROS is null " + + Thread.currentThread().getName(); + this.recordingOutputStream.write(b,off,count); + } + return count; + } + + public int read(byte[] b) throws IOException { + if (!isOpen()) { + throw new IOException("Stream closed " + + Thread.currentThread().getName()); + } + int count = this.in.read(b); + if (count > 0) { + assert this.recordingOutputStream != null: "ROS is null " + + Thread.currentThread().getName(); + this.recordingOutputStream.write(b,0,count); + } + return count; + } + + public void close() throws IOException { + if (logger.isLoggable(Level.FINE)) { + logger.fine(Thread.currentThread().getName() + " closing " + + this.in + ", " + Thread.currentThread().getName()); + } + if (this.in != null) { + this.in.close(); + this.in = null; + } + this.recordingOutputStream.close(); + } + + public ReplayInputStream getReplayInputStream() throws IOException { + return this.recordingOutputStream.getReplayInputStream(); + } + + public ReplayInputStream getContentReplayInputStream() throws IOException { + return this.recordingOutputStream.getContentReplayInputStream(); + } + + public long readFully() throws IOException { + while(read(drainBuffer) != -1) { + // Empty out stream. + continue; + } + return this.recordingOutputStream.getSize(); + } + + /** + * Read all of a stream (Or read until we timeout or have read to the max). + * @param softMaxLength Maximum length to read; if zero or < 0, then no + * limit. If met, return normally. + * @param hardMaxLength Maximum length to read; if zero or < 0, then no + * limit. If exceeded, throw RecorderLengthExceededException + * @param timeout Timeout in milliseconds for total read; if zero or + * negative, timeout is Long.MAX_VALUE. If exceeded, throw + * RecorderTimeoutException + * @param maxBytesPerMs How many bytes per millisecond. + * @throws IOException failed read. + * @throws RecorderLengthExceededException + * @throws RecorderTimeoutException + * @throws InterruptedException + */ + public void readFullyOrUntil(long softMaxLength) + throws IOException, RecorderLengthExceededException, + RecorderTimeoutException, InterruptedException { + // Check we're open before proceeding. + if (!isOpen()) { + // TODO: should this be a noisier exception-raising error? + return; + } + + long totalBytes = 0L; + long bytesRead = -1L; + long maxToRead = -1; + while (true) { + try { + // read no more than soft max + maxToRead = (softMaxLength <= 0) + ? drainBuffer.length + : Math.min(drainBuffer.length, softMaxLength - totalBytes); + // nor more than hard max + maxToRead = Math.min(maxToRead, recordingOutputStream.getRemainingLength()); + // but always at least 1 (to trigger hard max exception + maxToRead = Math.max(maxToRead, 1); + + bytesRead = read(drainBuffer,0,(int)maxToRead); + if (bytesRead == -1) { + break; + } + totalBytes += bytesRead; + + if (Thread.interrupted()) { + throw new InterruptedException("Interrupted during IO"); + } + } catch (SocketTimeoutException e) { + // A socket timeout is just a transient problem, meaning + // nothing was available in the configured timeout period, + // but something else might become available later. + // Take this opportunity to check the overall + // timeout (below). One reason for this timeout is + // servers that keep up the connection, 'keep-alive', even + // though we asked them to not keep the connection open. + if (logger.isLoggable(Level.FINE)) { + logger.log(Level.FINE, "socket timeout", e); + } + // check for overall timeout + recordingOutputStream.checkLimits(); + } catch (SocketException se) { + throw se; + } catch (NullPointerException e) { + // [ 896757 ] NPEs in Andy's Th-Fri Crawl. + // A crawl was showing NPE's in this part of the code but can + // not reproduce. Adding this rethrowing catch block w/ + // diagnostics to help should we come across the problem in the + // future. + throw new NullPointerException("Stream " + this.in + ", " + + e.getMessage() + " " + Thread.currentThread().getName()); + } + + // if have read 'enough', just finish + if (softMaxLength > 0 && totalBytes >= softMaxLength) { + break; // return + } + } + } + + public long getSize() { + return this.recordingOutputStream.getSize(); + } + + public void markContentBegin() { + this.recordingOutputStream.markContentBegin(); + } + + public long getContentBegin() { + return this.recordingOutputStream.getContentBegin(); + } + + public void startDigest() { + this.recordingOutputStream.startDigest(); + } + + /** + * Convenience method for setting SHA1 digest. + */ + public void setSha1Digest() { + this.recordingOutputStream.setSha1Digest(); + } + + /** + * Sets a digest algorithm which may be applied to recorded data. + * As usually only a subset of the recorded data should + * be fed to the digest, you must also call startDigest() + * to begin digesting. + * + * @param algorithm + */ + public void setDigest(String algorithm) { + this.recordingOutputStream.setDigest(algorithm); + } + + /** + * Sets a digest function which may be applied to recorded data. + * As usually only a subset of the recorded data should + * be fed to the digest, you must also call startDigest() + * to begin digesting. + * + * @param md + */ + public void setDigest(MessageDigest md) { + this.recordingOutputStream.setDigest(md); + } + + /** + * Return the digest value for any recorded, digested data. Call + * only after all data has been recorded; otherwise, the running + * digest state is ruined. + * + * @return the digest final value + */ + public byte[] getDigestValue() { + return this.recordingOutputStream.getDigestValue(); + } + + public ReplayCharSequence getReplayCharSequence() throws IOException { + return getReplayCharSequence(null); + } + + /** + * @param characterEncoding Encoding of recorded stream. + * @return A ReplayCharSequence Will return null if an IOException. Call + * close on returned RCS when done. + * @throws IOException + */ + public ReplayCharSequence getReplayCharSequence(String characterEncoding) + throws IOException { + return this.recordingOutputStream. + getReplayCharSequence(characterEncoding); + } + + public long getResponseContentLength() { + return this.recordingOutputStream.getResponseContentLength(); + } + + public void closeRecorder() throws IOException { + this.recordingOutputStream.closeRecorder(); + } + + /** + * @param tempFile + * @throws IOException + */ + public void copyContentBodyTo(File tempFile) throws IOException { + FileOutputStream fos = new FileOutputStream(tempFile); + ReplayInputStream ris = getContentReplayInputStream(); + ris.readFullyTo(fos); + fos.close(); + ris.close(); + } + + /** + * @return True if we've been opened. + */ + public boolean isOpen() + { + return this.in != null; + } + + @Override + public synchronized void mark(int readlimit) { + this.in.mark(readlimit); + this.recordingOutputStream.mark(); + } + + @Override + public boolean markSupported() { + return this.in.markSupported(); + } + + @Override + public synchronized void reset() throws IOException { + this.in.reset(); + this.recordingOutputStream.reset(); + } + + /** + * Set limits to be enforced by internal recording-out + */ + public void setLimits(long hardMax, long timeoutMs, long maxRateKBps) { + recordingOutputStream.setLimits(hardMax, timeoutMs, maxRateKBps); + } +} diff --git a/commons/src/main/java/org/archive/io/RecordingOutputStream.java b/commons/src/main/java/org/archive/io/RecordingOutputStream.java new file mode 100644 index 00000000..97e5c37f --- /dev/null +++ b/commons/src/main/java/org/archive/io/RecordingOutputStream.java @@ -0,0 +1,625 @@ +/* ReplayableOutputStream + * + * $Id$ + * + * Created on Sep 23, 2003 + * + * Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.io; + +import it.unimi.dsi.fastutil.io.FastBufferedOutputStream; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.archive.util.IoUtils; + + +/** + * An output stream that records all writes to wrapped output + * stream. + * + * A RecordingOutputStream can be wrapped around any other + * OutputStream to record all bytes written to it. You can + * then request a ReplayInputStream to read those bytes. + * + *

The RecordingOutputStream uses an in-memory buffer and + * backing disk file to allow it to record streams of + * arbitrary length limited only by available disk space. + * + *

As long as the stream recorded is smaller than the + * in-memory buffer, no disk access will occur. + * + *

Recorded content can be recovered as a ReplayInputStream + * (via getReplayInputStream() or, for only the content after + * the content-begin-mark is set, getContentReplayInputStream() ) + * or as a ReplayCharSequence (via getReplayCharSequence()). + * + *

This class is also used as a straight output stream + * by {@link RecordingInputStream} to which it records all reads. + * {@link RecordingInputStream} is exploiting the file backed buffer + * facility of this class passing null for the stream + * to wrap. TODO: Make a FileBackedOutputStream class that is + * subclassed by RecordingInputStream. + * + * @author gojomo + * + */ +public class RecordingOutputStream extends OutputStream { + protected static Logger logger = + Logger.getLogger(RecordingOutputStream.class.getName()); + + /** + * Size of recording. + * + * Later passed to ReplayInputStream on creation. It uses it to know when + * EOS. + */ + private long size = 0; + + private String backingFilename; + private OutputStream diskStream = null; + + /** + * Buffer we write recordings to. + * + * We write all recordings here first till its full. Thereafter we + * write the backing file. + */ + private byte[] buffer; + + /** current virtual position in the recording */ + private long position; + + /** flag to disable recording */ + private boolean recording; + + /** + * Reusable buffer for FastBufferedOutputStream + */ + protected byte[] bufStreamBuf = + new byte [ FastBufferedOutputStream.DEFAULT_BUFFER_SIZE ]; + + /** + * True if we're to digest content. + */ + private boolean shouldDigest = false; + + /** + * Digest instance. + */ + private MessageDigest digest = null; + + /** + * Define for SHA1 alogarithm. + */ + private static final String SHA1 = "SHA1"; + + /** + * Maximum amount of header material to accept without the content + * body beginning -- if more, throw a RecorderTooMuchHeaderException. + * TODO: make configurable? make smaller? + */ + protected static final long MAX_HEADER_MATERIAL = 1024*1024; // 1MB + + // configurable max length, max time limits + /** maximum length of material to record before throwing exception */ + protected long maxLength = Long.MAX_VALUE; + /** maximum time to record before throwing exception */ + protected long timeoutMs = Long.MAX_VALUE; + /** maximum rate to record (adds delays to hit target rate) */ + protected long maxRateBytesPerMs = Long.MAX_VALUE; + /** time recording begins for timeout, rate calculations */ + protected long startTime = Long.MAX_VALUE; + + /** + * When recording HTTP, where the content-body starts. + */ + private long contentBeginMark; + + /** + * Stream to record. + */ + private OutputStream out = null; + + // mark/reset support + /** furthest position reached before any reset()s */ + private long maxPosition = 0; + /** remembered position to reset() to */ + private long markPosition = 0; + + /** + * Create a new RecordingOutputStream. + * + * @param bufferSize Buffer size to use. + * @param backingFilename Name of backing file to use. + */ + public RecordingOutputStream(int bufferSize, String backingFilename) { + this.buffer = new byte[bufferSize]; + this.backingFilename = backingFilename; + recording = true; + } + + /** + * Wrap the given stream, both recording and passing along any data written + * to this RecordingOutputStream. + * + * @throws IOException If failed creation of backing file. + */ + public void open() throws IOException { + this.open(null); + } + + /** + * Wrap the given stream, both recording and passing along any data written + * to this RecordingOutputStream. + * + * @param wrappedStream Stream to wrap. May be null for case where we + * want to write to a file backed stream only. + * + * @throws IOException If failed creation of backing file. + */ + public void open(OutputStream wrappedStream) throws IOException { + if(isOpen()) { + // error; should not be opening/wrapping in an unclosed + // stream remains open + throw new IOException("ROS already open for " + +Thread.currentThread().getName()); + } + this.out = wrappedStream; + this.position = 0; + this.markPosition = 0; + this.maxPosition = 0; + this.size = 0; + this.contentBeginMark = -1; + // ensure recording turned on + this.recording = true; + // Always begins false; must use startDigest() to begin + this.shouldDigest = false; + if (this.diskStream != null) { + closeDiskStream(); + } + if (this.diskStream == null) { + // TODO: Fix so we only make file when its actually needed. + FileOutputStream fis = new FileOutputStream(this.backingFilename); + + this.diskStream = new RecyclingFastBufferedOutputStream(fis, bufStreamBuf); + } + startTime = System.currentTimeMillis(); + } + + public void write(int b) throws IOException { + if(positionMAX_HEADER_MATERIAL) { + throw new RecorderTooMuchHeaderException(); + } + } + // overlong? + if(position>maxLength) { + throw new RecorderLengthExceededException(); + } + // taking too long? + long duration = System.currentTimeMillis() - startTime; + duration = Math.max(duration,1); // !divzero + if(duration>timeoutMs) { + throw new RecorderTimeoutException(); + } + // need to throttle reading to hit max configured rate? + if(position/duration > maxRateBytesPerMs) { + long desiredDuration = position / maxRateBytesPerMs; + try { + Thread.sleep(desiredDuration-duration); + } catch (InterruptedException e) { + logger.log(Level.WARNING, + "bandwidth throttling sleep interrupted", e); + } + } + } + + /** + * Record the given byte for later recovery + * + * @param b Int to record. + * + * @exception IOException Failed write to backing file. + */ + private void record(int b) throws IOException { + if (this.shouldDigest) { + this.digest.update((byte)b); + } + if (this.position >= this.buffer.length) { + // TODO: Its possible to call write w/o having first opened a + // stream. Protect ourselves against this. + assert this.diskStream != null: "Diskstream is null"; + this.diskStream.write(b); + } else { + this.buffer[(int) this.position] = (byte) b; + } + this.position++; + } + + /** + * Record the given byte-array range for recovery later + * + * @param b Buffer to record. + * @param off Offset into buffer at which to start recording. + * @param len Length of buffer to record. + * + * @exception IOException Failed write to backing file. + */ + private void record(byte[] b, int off, int len) throws IOException { + if(this.shouldDigest) { + assert this.digest != null: "Digest is null."; + this.digest.update(b, off, len); + } + tailRecord(b, off, len); + } + + /** + * Record without digesting. + * + * @param b Buffer to record. + * @param off Offset into buffer at which to start recording. + * @param len Length of buffer to record. + * + * @exception IOException Failed write to backing file. + */ + private void tailRecord(byte[] b, int off, int len) throws IOException { + if(this.position >= this.buffer.length){ + // TODO: Its possible to call write w/o having first opened a + // stream. Lets protect ourselves against this. + if (this.diskStream == null) { + throw new IOException("diskstream is null"); + } + this.diskStream.write(b, off, len); + this.position += len; + } else { + assert this.buffer != null: "Buffer is null"; + int toCopy = (int)Math.min(this.buffer.length - this.position, len); + assert b != null: "Passed buffer is null"; + System.arraycopy(b, off, this.buffer, (int)this.position, toCopy); + this.position += toCopy; + // TODO verify these are +1 -1 right + if (toCopy < len) { + tailRecord(b, off + toCopy, len - toCopy); + } + } + } + + public void close() throws IOException { + if(contentBeginMark<0) { + // if unset, consider 0 posn as content-start + // (so that a -1 never survives to replay step) + contentBeginMark = 0; + } + if (this.out != null) { + this.out.close(); + this.out = null; + } + closeRecorder(); + } + + protected synchronized void closeDiskStream() + throws IOException { + if (this.diskStream != null) { + this.diskStream.close(); + this.diskStream = null; + } + } + + public void closeRecorder() throws IOException { + recording = false; + closeDiskStream(); // if any + // This setting of size is important. Its passed to ReplayInputStream + // on creation. It uses it to know EOS. + if (this.size == 0) { + this.size = this.position; + } + } + + /* (non-Javadoc) + * @see java.io.OutputStream#flush() + */ + public void flush() throws IOException { + if (this.out != null) { + this.out.flush(); + } + if (this.diskStream != null) { + this.diskStream.flush(); + } + } + + public ReplayInputStream getReplayInputStream() throws IOException { + return getReplayInputStream(0); + } + + public ReplayInputStream getReplayInputStream(long skip) throws IOException { + // If this method is being called, then assumption must be that the + // stream is closed. If it ain't, then the stream gotten won't work + // -- the size will zero so any attempt at a read will get back EOF. + assert this.out == null: "Stream is still open."; + ReplayInputStream replay = new ReplayInputStream(this.buffer, + this.size, this.contentBeginMark, this.backingFilename); + replay.skip(skip); + return replay; + } + + /** + * Return a replay stream, cued up to begining of content + * + * @throws IOException + * @return An RIS. + */ + public ReplayInputStream getContentReplayInputStream() throws IOException { + return getReplayInputStream(this.contentBeginMark); + } + + public long getSize() { + return this.size; + } + + /** + * Remember the current position as the start of the "response + * body". Useful when recording HTTP traffic as a way to start + * replays after the headers. + */ + public void markContentBegin() { + this.contentBeginMark = this.position; + startDigest(); + } + + /** + * Return stored content-begin-mark (which is also end-of-headers) + */ + public long getContentBegin() { + return this.contentBeginMark; + } + + /** + * Starts digesting recorded data, if a MessageDigest has been + * set. + */ + public void startDigest() { + if (this.digest != null) { + this.digest.reset(); + this.shouldDigest = true; + } + } + + /** + * Convenience method for setting SHA1 digest. + * @see #setDigest(String) + */ + public void setSha1Digest() { + setDigest(SHA1); + } + + + /** + * Sets a digest function which may be applied to recorded data. + * The difference between calling this method and {@link #setDigest(MessageDigest)} + * is that this method tries to reuse MethodDigest instance if already allocated + * and of appropriate algorithm. + * @param algorithm Message digest algorithm to use. + * @see #setDigest(MessageDigest) + */ + public void setDigest(String algorithm) { + try { + // Reuse extant digest if its sha1 algorithm. + if (this.digest == null || + !this.digest.getAlgorithm().equals(algorithm)) { + setDigest(MessageDigest.getInstance(algorithm)); + } + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + } + } + + /** + * Sets a digest function which may be applied to recorded data. + * + * As usually only a subset of the recorded data should + * be fed to the digest, you must also call startDigest() + * to begin digesting. + * + * @param md Message digest function to use. + */ + public void setDigest(MessageDigest md) { + this.digest = md; + } + + /** + * Return the digest value for any recorded, digested data. Call + * only after all data has been recorded; otherwise, the running + * digest state is ruined. + * + * @return the digest final value + */ + public byte[] getDigestValue() { + if(this.digest == null) { + return null; + } + return this.digest.digest(); + } + + public ReplayCharSequence getReplayCharSequence() throws IOException { + return getReplayCharSequence(null); + } + + public ReplayCharSequence getReplayCharSequence(String characterEncoding) + throws IOException { + return getReplayCharSequence(characterEncoding, this.contentBeginMark); + } + + /** + * @param characterEncoding Encoding of recorded stream. + * @return A ReplayCharSequence Will return null if an IOException. Call + * close on returned RCS when done. + * @throws IOException + */ + public ReplayCharSequence getReplayCharSequence(String characterEncoding, + long startOffset) throws IOException { + // TODO: handled transfer-encoding: chunked content-bodies properly + float maxBytesPerChar = IoUtils.encodingMaxBytesPerChar(characterEncoding); + if(maxBytesPerChar<=1) { + // single + // TODO: take into account single-byte encoding may be non-default + return new ByteReplayCharSequence( + this.buffer, + this.size, + startOffset, + this.backingFilename); + } else { + // multibyte + if(this.size <= this.buffer.length) { + // raw data is all in memory; do in memory + return new MultiByteReplayCharSequence( + this.buffer, + this.size, + startOffset, + characterEncoding); + + } else { + // raw data overflows to disk; use temp file + ReplayInputStream ris = getReplayInputStream(startOffset); + ReplayCharSequence rcs = new MultiByteReplayCharSequence( + ris, + this.backingFilename, + characterEncoding); + ris.close(); + return rcs; + } + + } + + } + + public long getResponseContentLength() { + return this.size - this.contentBeginMark; + } + + /** + * @return True if this ROS is open. + */ + public boolean isOpen() { + return this.out != null; + } + + /** + * When used alongside a mark-supporting RecordingInputStream, remember + * a position reachable by a future reset(). + */ + public void mark() { + // remember this position for subsequent reset() + this.markPosition = position; + } + + /** + * When used alongside a mark-supporting RecordingInputStream, reset + * the position to that saved by previous mark(). Until the position + * again reached "new" material, none of the bytes pushed to this + * stream will be digested or recorded. + */ + public void reset() { + // take note of furthest-position-reached to avoid double-recording + maxPosition = Math.max(maxPosition, position); + // reset to previous position + position = markPosition; + } + + /** + * Set limits on length, time, and rate to enforce. + * + * @param length + * @param milliseconds + * @param rateKBps + */ + public void setLimits(long length, long milliseconds, long rateKBps) { + maxLength = (length>0) ? length : Long.MAX_VALUE; + timeoutMs = (milliseconds>0) ? milliseconds : Long.MAX_VALUE; + maxRateBytesPerMs = (rateKBps>0) ? rateKBps*1024/1000 : Long.MAX_VALUE; + } + + /** + * Reset limits to effectively-unlimited defaults + */ + public void resetLimits() { + maxLength = Long.MAX_VALUE; + timeoutMs = Long.MAX_VALUE; + maxRateBytesPerMs = Long.MAX_VALUE; + } + + /** + * Return number of bytes that could be recorded without hitting + * length limit + * + * @return long byte count + */ + public long getRemainingLength() { + return maxLength - position; + } +} diff --git a/commons/src/main/java/org/archive/io/RecoverableIOException.java b/commons/src/main/java/org/archive/io/RecoverableIOException.java new file mode 100644 index 00000000..c4cf7b55 --- /dev/null +++ b/commons/src/main/java/org/archive/io/RecoverableIOException.java @@ -0,0 +1,87 @@ +/* $Id$ + * + * Created on August 21st, 2006 + * + * Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.io; + +import java.io.IOException; +import java.io.PrintStream; +import java.io.PrintWriter; + +/** + * A decorator on IOException for IOEs that are likely not fatal or at least + * merit retry. + * @author stack + * @version $Date$, $Revision$ + */ +public class RecoverableIOException extends IOException { + private static final long serialVersionUID = 6194776587381865451L; + private final IOException decoratedIOException; + + public RecoverableIOException(final String message) { + this(new IOException(message)); + } + + public RecoverableIOException(final IOException ioe) { + super(); + this.decoratedIOException = ioe; + } + + public Throwable getCause() { + return this.decoratedIOException.getCause(); + } + + public String getLocalizedMessage() { + return this.decoratedIOException.getLocalizedMessage(); + } + + public String getMessage() { + return this.decoratedIOException.getMessage(); + } + + public StackTraceElement[] getStackTrace() { + return this.decoratedIOException.getStackTrace(); + } + + public synchronized Throwable initCause(Throwable cause) { + return this.decoratedIOException.initCause(cause); + } + + public void printStackTrace() { + this.decoratedIOException.printStackTrace(); + } + + public void printStackTrace(PrintStream s) { + this.decoratedIOException.printStackTrace(s); + } + + public void printStackTrace(PrintWriter s) { + this.decoratedIOException.printStackTrace(s); + } + + public void setStackTrace(StackTraceElement[] stackTrace) { + this.decoratedIOException.setStackTrace(stackTrace); + } + + public String toString() { + return this.decoratedIOException.toString(); + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/io/RecyclingFastBufferedOutputStream.java b/commons/src/main/java/org/archive/io/RecyclingFastBufferedOutputStream.java new file mode 100644 index 00000000..6ea3925f --- /dev/null +++ b/commons/src/main/java/org/archive/io/RecyclingFastBufferedOutputStream.java @@ -0,0 +1,150 @@ +/* RecyclingFastBufferedOutputStream +* +* $Id$ +* +* Created on May 26, 2005 +* +* Based on FastBufferedOutputStream in MG4J; see: +* +* http://mg4j.dsi.unimi.it/ +* +* (Sole addition is one new constructor.) +* +* Revisions copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.io; + +import java.io.IOException; +import java.io.OutputStream; + +/** Lightweight, unsynchronised, aligned output stream buffering class. + * + *

This class provides buffering for output streams, but it does so with + * purposes and an internal logic that are radically different from the ones + * adopted in {@link java.io.BufferedOutputStream}. + * + *

All methods are unsychronised. Moreover, + * it is guaranteed that all writes performed by this class will be + * multiples of the given buffer size. If, for instance, you use the + * default buffer size, writes will be performed on the underlying input stream + * in multiples of 16384 bytes. This is very important on operating systems + * that optimise disk reads on disk block boundaries. + */ + +public class RecyclingFastBufferedOutputStream extends OutputStream { + + /** The default size of the internal buffer in bytes (16Ki). */ + public final static int DEFAULT_BUFFER_SIZE = 16 * 1024; + + /** The internal buffer. */ + protected byte buffer[]; + + /** The current position in the buffer. */ + protected int pos; + + /** The number of buffer bytes available starting from {@link #pos}. */ + protected int avail; + + /** The underlying output stream. */ + protected OutputStream os; + + /** Creates a new fast buffered output stream by wrapping a given output stream, using a given buffer + * + * @param os an output stream to wrap. + * @param buffer buffer to use internally. + */ + + public RecyclingFastBufferedOutputStream( final OutputStream os, final byte[] buffer ) { + this.os = os; + this.buffer = buffer; + avail = buffer.length; + } + + /** Creates a new fast buffered output stream by wrapping a given output stream with a given buffer size. + * + * @param os an output stream to wrap. + * @param bufSize the size in bytes of the internal buffer. + */ + + public RecyclingFastBufferedOutputStream( final OutputStream os, final int bufSize ) { + this(os, new byte [ bufSize]); + } + + /** Creates a new fast buffered ouptut stream by wrapping a given output stream with a buffer of {@link #DEFAULT_BUFFER_SIZE} bytes. + * + * @param os an output stream to wrap. + */ + public RecyclingFastBufferedOutputStream( final OutputStream os ) { + this( os, DEFAULT_BUFFER_SIZE ); + } + + private void dumpBufferIfFull() throws IOException { + if ( avail == 0 ) { + os.write( buffer ); + pos = 0; + avail = buffer.length; + } + } + + public void write( final int b ) throws IOException { + avail--; + buffer[ pos++ ] = (byte)b; + dumpBufferIfFull(); + } + + + public void write( final byte b[], int offset, int length ) throws IOException { + if ( length <= avail ) { + System.arraycopy( b, offset, buffer, pos, length ); + pos += length; + avail -= length; + dumpBufferIfFull(); + return; + } + + System.arraycopy( b, offset, buffer, pos, avail ); + os.write( buffer ); + + offset += avail; + length -= avail; + + final int residual = length % buffer.length; + + os.write( b, offset, length - residual ); + System.arraycopy( b, offset + length - residual, buffer, 0, residual ); + pos = residual; + avail = buffer.length - residual; + } + + public void close() throws IOException { + if ( os == null ) return; + if ( pos != 0 ) os.write( buffer, 0, pos ); + if ( os != System.out ) os.close(); + os = null; + buffer = null; + } + +} + + +// Local Variables: +// mode: jde +// tab-width: 4 +// End: + diff --git a/commons/src/main/java/org/archive/io/ReplayCharSequence.java b/commons/src/main/java/org/archive/io/ReplayCharSequence.java new file mode 100644 index 00000000..f9d8c83f --- /dev/null +++ b/commons/src/main/java/org/archive/io/ReplayCharSequence.java @@ -0,0 +1,46 @@ +/* ReplayCharSequence + * + * Created on Mar 5, 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.io; + +import java.io.IOException; + + +/** + * CharSequence interface with addition of a {@link #close()} method. + * + * Users of implementations of this interface must call {@link #close()} so + * implementations get a chance at cleaning up after themselves. + * + * @author stack + * @version $Revision$, $Date$ + */ +public interface ReplayCharSequence extends CharSequence { + + /** + * Call this method when done so implementation has chance to clean up + * resources. + * + * @throws IOException Problem cleaning up file system resources. + */ + public void close() throws IOException; +} diff --git a/commons/src/main/java/org/archive/io/ReplayInputStream.java b/commons/src/main/java/org/archive/io/ReplayInputStream.java new file mode 100644 index 00000000..a590ccdb --- /dev/null +++ b/commons/src/main/java/org/archive/io/ReplayInputStream.java @@ -0,0 +1,282 @@ +/* ReplayInputStream + * + * $Id$ + * + * Created on Sep 24, 2003 + * + * Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.io; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; + + +/** + * Replays the bytes recorded from a RecordingInputStream or + * RecordingOutputStream. + * + * This InputStream supports mark and reset. + * + * @author gojomo + */ +public class ReplayInputStream extends SeekInputStream +{ + private BufferedSeekInputStream diskStream; + private byte[] buffer; + private long position; + + /** + * Total size of stream content. + * + * Size of data to replay. + */ + private long size = -1; + + /** + * Where the response body starts, if marked + */ + protected long responseBodyStart = -1; + + + /** + * Constructor. + * + * @param buffer Buffer to read from. + * @param size Size of data to replay. + * @param responseBodyStart Start of the response body. + * @param backingFilename Backing file that sits behind the buffer. If + * size > than buffer then we go to backing file to read + * data that is beyond buffer.length. + * + * @throws IOException If we fail to open an input stream on + * backing file. + */ + public ReplayInputStream(byte[] buffer, long size, long responseBodyStart, + String backingFilename) + throws IOException + { + this(buffer, size, backingFilename); + this.responseBodyStart = responseBodyStart; + } + + /** + * Constructor. + * + * @param buffer Buffer to read from. + * @param size Size of data to replay. + * @param backingFilename Backing file that sits behind the buffer. If + * size > than buffer then we go to backing file to read + * data that is beyond buffer.length. + * @throws IOException If we fail to open an input stream on + * backing file. + */ + public ReplayInputStream(byte[] buffer, long size, String backingFilename) + throws IOException + { + this.buffer = buffer; + this.size = size; + if (size > buffer.length) { + RandomAccessInputStream rais = new RandomAccessInputStream( + new File(backingFilename)); + diskStream = new BufferedSeekInputStream(rais, 4096); + } + } + + public long setToResponseBodyStart() throws IOException { + position(responseBodyStart); + return this.position; + } + + + /* (non-Javadoc) + * @see java.io.InputStream#read() + */ + public int read() throws IOException { + if (position == size) { + return -1; // EOF + } + if (position < buffer.length) { + // Convert to unsigned int. + int c = buffer[(int) position] & 0xFF; + position++; + return c; + } + int c = diskStream.read(); + if (c >= 0) { + position++; + } + return c; + } + + /* + * (non-Javadoc) + * + * @see java.io.InputStream#read(byte[], int, int) + */ + public int read(byte[] b, int off, int len) throws IOException { + if (position == size) { + return -1; // EOF + } + if (position < buffer.length) { + int toCopy = (int)Math.min(size - position, + Math.min(len, buffer.length - position)); + System.arraycopy(buffer, (int)position, b, off, toCopy); + if (toCopy > 0) { + position += toCopy; + } + return toCopy; + } + // into disk zone + int read = diskStream.read(b,off,len); + if(read>0) { + position += read; + } + return read; + } + + public void readFullyTo(OutputStream os) throws IOException { + byte[] buf = new byte[4096]; + int c = read(buf); + while (c != -1) { + os.write(buf,0,c); + c = read(buf); + } + } + + /* + * Like 'readFullyTo', but only reads the header-part. + * Starts from the beginning each time it is called. + */ + public void readHeaderTo(OutputStream os) throws IOException { + position = 0; + byte[] buf = new byte[(int)responseBodyStart]; + int c = read(buf,0,buf.length); + if(c != -1) { + os.write(buf,0,c); + } + } + + /* + * Like 'readFullyTo', but only reads the content-part. + */ + public void readContentTo(OutputStream os) throws IOException { + setToResponseBodyStart(); + byte[] buf = new byte[4096]; + int c = read(buf); + while (c != -1) { + os.write(buf,0,c); + c = read(buf); + } + } + + /** + * Convenience method to copy content out to target stream. + * @param os stream to write content to + * @param maxSize maximum count of bytes to copy + * @throws IOException + */ + public void readContentTo(OutputStream os, long maxSize) throws IOException { + setToResponseBodyStart(); + byte[] buf = new byte[4096]; + int c = read(buf); + long tot = 0; + while (c != -1 && tot < maxSize) { + os.write(buf,0,c); + c = read(buf); + tot += c; + } + } + + /* (non-Javadoc) + * @see java.io.InputStream#close() + */ + public void close() throws IOException { + super.close(); + if(diskStream != null) { + diskStream.close(); + } + } + + /** + * Total size of stream content. + * @return Returns the size. + */ + public long getSize() + { + return size; + } + + /** + * Total size of header. + * @return the size of the header. + */ + public long getHeaderSize() + { + return responseBodyStart; + } + + /** + * Total size of content. + * @return the size of the content. + */ + public long getContentSize() + { + return size - responseBodyStart; + } + + /** + * @return Amount THEORETICALLY remaining (TODO: Its not theoretical + * seemingly. The class implemetentation depends on it being exact). + */ + public long remaining() { + return size - position; + } + + + /** + * Reposition the stream. + * + * @param p the new position for this stream + * @throws IOException if an IO error occurs + */ + public void position(long p) throws IOException { + if (p < 0) { + throw new IOException("Negative seek offset."); + } + if (p > size) { + throw new IOException("Desired position exceeds size."); + } + if (p < buffer.length) { + // Only seek file if necessary + if (position > buffer.length) { + diskStream.position(0); + } + } else { + diskStream.position(p - buffer.length); + } + this.position = p; + } + + + public long position() throws IOException { + return position; + } +} diff --git a/commons/src/main/java/org/archive/io/RepositionableInputStream.java b/commons/src/main/java/org/archive/io/RepositionableInputStream.java new file mode 100644 index 00000000..f5fe6b52 --- /dev/null +++ b/commons/src/main/java/org/archive/io/RepositionableInputStream.java @@ -0,0 +1,139 @@ +/* RepositionableInputStream.java + * + * $Id$ + * + * Created Dec 20, 2005 + * + * Copyright (C) 2005 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.io; + +import it.unimi.dsi.fastutil.io.RepositionableStream; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; + +/** + * Wrapper around an {@link InputStream} to make a primitive Repositionable + * stream. Uses a {@link BufferedInputStream}. Calls mark on every read so + * we'll remember at least the last thing read (You can only backup on the + * last thing read -- not last 2 or 3 things read). Used by + * {@link GzippedInputStream} when reading streams over a network. Wraps a + * HTTP, etc., stream so we can back it up if needs be after the + * GZIP inflater has done a fill of its full buffer though it only needed + * the first few bytes to finish decompressing the current GZIP member. + * + *

TODO: More robust implementation. Tried to use the it.unimi.dsi.io + * FastBufferdInputStream but relies on FileChannel ByteBuffers and if not + * present -- as would be the case reading from a network stream, the main + * application for this instance -- then it expects the underlying stream + * implements RepositionableStream interface so chicken or egg problem. + * @author stack + */ +public class RepositionableInputStream extends BufferedInputStream implements + RepositionableStream { + private long position = 0; + private long markPosition = -1; + + public RepositionableInputStream(InputStream in) { + super(in); + } + + public RepositionableInputStream(InputStream in, int size) { + super(in, size); + } + + public int read(byte[] b) throws IOException { + int read = super.read(b); + if (read != -1) { + position += read; + } + return read; + } + + public synchronized int read(byte[] b, int offset, int ct) + throws IOException { + // Mark the underlying stream so that we'll remember what we are about + // to read unless a mark has been set in this RepositionableStream + // (We have two levels of mark). In this latter case we want the + // underlying stream to preserve its mark position so aligns with + // this RS when eset is called. + if (!isMarked()) { + super.mark((ct > offset)? ct - offset: ct); + } + int read = super.read(b, offset, ct); + if (read != -1) { + position += read; + } + return read; + } + + public int read() throws IOException { + // Mark the underlying stream so that we'll remember what we are about + // to read unless a mark has been set in this RepositionableStream + // (We have two levels of mark). In this latter case we want the + // underlying stream to preserve its mark position so aligns with + // this RS when eset is called. + if (!isMarked()) { + super.mark(1); + } + int c = super.read(); + if (c != -1) { + position++; + } + return c; + } + + public void position(final long offset) { + if (this.position == offset) { + return; + } + int diff = (int)(offset - this.position); + long lowerBound = this.position - this.pos; + long upperBound = lowerBound + this.count; + if (offset < lowerBound || offset >= upperBound) { + throw new IllegalAccessError("Offset goes outside " + + "current this.buf (TODO: Do buffer fills if positive)"); + } + this.position = offset; + this.pos += diff; + // Clear any mark. + this.markPosition = -1; + } + + public void mark(int readlimit) { + this.markPosition = this.position; + super.mark(readlimit); + } + + public void reset() throws IOException { + super.reset(); + this.position = this.markPosition; + this.markPosition = -1; + } + + protected boolean isMarked() { + return this.markPosition != -1; + } + + public long position() { + return this.position; + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/io/SafeSeekInputStream.java b/commons/src/main/java/org/archive/io/SafeSeekInputStream.java new file mode 100644 index 00000000..a40a08bf --- /dev/null +++ b/commons/src/main/java/org/archive/io/SafeSeekInputStream.java @@ -0,0 +1,128 @@ +/* SafeSeekInputStream +* +* Created on September 14, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.io; + + +import java.io.IOException; + + +/** + * Enables multiple concurrent streams based on the same underlying stream. + * + * @author pjack + */ +public class SafeSeekInputStream extends SeekInputStream { + + + /** + * The underlying stream. + */ + private SeekInputStream input; + + + /** + * The expected position of the underlying stream. + */ + private long expected; + + + /** + * Constructor. The given stream will be positioned to 0 so that an + * accurate position can be tracked. + * + * @param input the underlying input stream + * @throws IOException if an IO error occurs + */ + public SafeSeekInputStream(SeekInputStream input) throws IOException { + this.input = input; + this.expected = input.position(); + } + + + /** + * Ensures that the underlying stream's position is what we expect to be. + * + * @throws IOException if an IO error occurs + */ + private void ensure() throws IOException { + if (expected != input.position()) { + input.position(expected); + } + } + + + @Override + public int read() throws IOException { + ensure(); + int c = input.read(); + if (c >= 0) { + expected++; + } + return c; + } + + + @Override + public int read(byte[] buf, int ofs, int len) throws IOException { + ensure(); + int r = input.read(buf, ofs, len); + if (r > 0) { + expected += r; + } + return r; + } + + + @Override + public int read(byte[] buf) throws IOException { + ensure(); + int r = input.read(buf); + if (r > 0) { + expected += r; + } + return r; + } + + + @Override + public long skip(long c) throws IOException { + ensure(); + long r = input.skip(c); + if (r > 0) { + expected += r; + } + return r; + } + + + public void position(long p) throws IOException { + input.position(p); + expected = p; + } + + + public long position() throws IOException { + return expected; + } + +} diff --git a/commons/src/main/java/org/archive/io/SeekInputStream.java b/commons/src/main/java/org/archive/io/SeekInputStream.java new file mode 100644 index 00000000..c46405c8 --- /dev/null +++ b/commons/src/main/java/org/archive/io/SeekInputStream.java @@ -0,0 +1,85 @@ +/* SeekInputStream +* +* Created on September 12, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.io; + + +import it.unimi.dsi.fastutil.io.RepositionableStream; + +import java.io.IOException; +import java.io.InputStream; + + +/** + * Base class for repositionable input streams. + * + * @author pjack + */ +public abstract class SeekInputStream extends InputStream +implements RepositionableStream { + + + /** + * The marked file position. A value less than zero + * indicates that no mark has been set. + */ + private long mark = -1; + + + /** + * Marks the current position of the stream. The limit parameter is + * ignored; the mark will remain valid until reset is called or the + * stream is closed. + * + * @param limit ignored + */ + public void mark(int limit) { + try { + this.mark = position(); + } catch (IOException e) { + mark = -1; + } + } + + + /** + * Resets this stream to its marked position. + * + * @throws IOException if there is no mark, or if an IO error occurs + */ + public void reset() throws IOException { + if (mark < 0) { + throw new IOException("No mark."); + } + position(mark); + } + + + /** + * Returns true, since SeekInputStreams support mark/reset by default. + * + * @return true + */ + public boolean markSupported() { + return true; + } +} diff --git a/commons/src/main/java/org/archive/io/SeekReader.java b/commons/src/main/java/org/archive/io/SeekReader.java new file mode 100644 index 00000000..c019aa27 --- /dev/null +++ b/commons/src/main/java/org/archive/io/SeekReader.java @@ -0,0 +1,88 @@ +/* SeekReader +* +* Created on September 18, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.io; + + +import java.io.IOException; +import java.io.Reader; + +import it.unimi.dsi.fastutil.io.RepositionableStream; + + +/** + * Base class for repositionable readers. + * + * @author pjack + */ +public abstract class SeekReader extends Reader +implements RepositionableStream { + + + /** + * The marked file position. A value less than zero + * indicates that no mark has been set. + */ + private long mark = -1; + + + /** + * Marks the current position of the stream. The limit parameter is + * ignored; the mark will remain valid until reset is called or the + * stream is closed. + * + * @param limit ignored + */ + @Override + public void mark(int limit) { + try { + this.mark = position(); + } catch (IOException e) { + mark = -1; + } + } + + + /** + * Resets this stream to its marked position. + * + * @throws IOException if there is no mark, or if an IO error occurs + */ + @Override + public void reset() throws IOException { + if (mark < 0) { + throw new IOException("No mark."); + } + position(mark); + } + + + /** + * Returns true, since SeekInputStreams support mark/reset by default. + * + * @return true + */ + @Override + public boolean markSupported() { + return true; + } +} diff --git a/commons/src/main/java/org/archive/io/SeekReaderCharSequence.java b/commons/src/main/java/org/archive/io/SeekReaderCharSequence.java new file mode 100644 index 00000000..a9b4880f --- /dev/null +++ b/commons/src/main/java/org/archive/io/SeekReaderCharSequence.java @@ -0,0 +1,56 @@ +package org.archive.io; + +import java.io.IOException; + +public class SeekReaderCharSequence implements CharSequence { + + + final private SeekReader reader; + final private int size; + + + public SeekReaderCharSequence(SeekReader reader, int size) { + this.reader = reader; + this.size = size; + } + + + public int length() { + return size; + } + + + public char charAt(int index) { + if ((index < 0) || (index >= length())) { + throw new IndexOutOfBoundsException(Integer.toString(index)); + } + try { + reader.position(index); + int r = reader.read(); + if (r < 0) { + throw new IllegalStateException("EOF"); + } + return (char)reader.read(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + + public CharSequence subSequence(int start, int end) { + return new CharSubSequence(this, start, end); + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + try { + reader.position(0); + for (int ch = reader.read(); ch >= 0; ch = reader.read()) { + sb.append((char)ch); + } + return sb.toString(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } +} diff --git a/commons/src/main/java/org/archive/io/SinkHandlerLogThread.java b/commons/src/main/java/org/archive/io/SinkHandlerLogThread.java new file mode 100644 index 00000000..89a899f7 --- /dev/null +++ b/commons/src/main/java/org/archive/io/SinkHandlerLogThread.java @@ -0,0 +1,38 @@ +/* SinkHandlerLogRecord + * + * Created Aug 9, 2005 + * + * Copyright (C) 2005 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +package org.archive.io; + + +/** + * Implemented by threads that provide extra information. + * + * TODO: rename class, rename getCurrentProcessorName() + */ +public interface SinkHandlerLogThread { + + String getName(); + String getCurrentProcessorName(); + int getSerialNumber(); + +} diff --git a/commons/src/main/java/org/archive/io/UTF8Bytes.java b/commons/src/main/java/org/archive/io/UTF8Bytes.java new file mode 100644 index 00000000..be326d5a --- /dev/null +++ b/commons/src/main/java/org/archive/io/UTF8Bytes.java @@ -0,0 +1,41 @@ +/* $Id$ + * + * Created Jul 27, 2006 + * + * Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.io; + +import java.io.UnsupportedEncodingException; + +/** + * Marker Interface for instances that can be serialized as UTF8 bytes. + * TODO: Do we need a UTF8Stream Marker Interface? + * @author stack + * @version $Date$ $Version$ + */ +public interface UTF8Bytes { + public static final String UTF8 = "UTF-8"; + + /** + * @return Instance as UTF-8 bytes. + * @throws UnsupportedEncodingException + */ + public byte [] getUTF8Bytes() throws UnsupportedEncodingException; +} diff --git a/commons/src/main/java/org/archive/io/Warc2Arc.java b/commons/src/main/java/org/archive/io/Warc2Arc.java new file mode 100644 index 00000000..d0b9377a --- /dev/null +++ b/commons/src/main/java/org/archive/io/Warc2Arc.java @@ -0,0 +1,226 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; +import org.archive.io.arc.ARCWriter; +import org.archive.io.warc.WARCConstants; +import org.archive.io.warc.WARCReader; +import org.archive.io.warc.WARCReaderFactory; +import org.archive.io.warc.WARCRecord; +import org.archive.util.ArchiveUtils; +import org.archive.util.FileUtils; + +/** + * Convert WARCs to (sortof) ARCs. + * WARCs can be 1Gig in size, that is, 10x default ARC size. Script takes + * directory as output and will write multiple ARCs for a single large WARC. + * Only writes resource records of type text/dns or + * application/http; msgtype=response. All others -- metadata, + * request -- are skipped. + * @author stack + * @version $Date$ $Revision$ + */ +public class Warc2Arc { + private static void usage(HelpFormatter formatter, Options options, + int exitCode) { + formatter.printHelp("java org.archive.io.arc.Warc2Arc " + + "[--force] [--prefix=PREFIX] [--suffix=SUFFIX] WARC_INPUT " + + "OUTPUT_DIR", + options); + System.exit(exitCode); + } + + static String parseRevision(final String version) { + final String ID = "$Revision: "; + int index = version.indexOf(ID); + return (index < 0)? version: + version.substring(index + ID.length(), version.length() - 1).trim(); + } + + private static String getRevision() { + return parseRevision("$Revision$"); + } + + public void transform(final File warc, final File dir, final String prefix, + final String suffix, final boolean force) + throws IOException, java.text.ParseException { + FileUtils.assertReadable(warc); + FileUtils.assertReadable(dir); + WARCReader reader = WARCReaderFactory.get(warc); + List metadata = new ArrayList(); + metadata.add("Made from " + reader.getReaderIdentifier() + " by " + + this.getClass().getName() + "/" + getRevision()); + ARCWriter writer = new ARCWriter(new AtomicInteger(), + Arrays.asList(new File [] {dir}), prefix, suffix, + reader.isCompressed(), -1, metadata); + transform(reader, writer); + } + + protected void transform(final WARCReader reader, final ARCWriter writer) + throws IOException, java.text.ParseException { + // No point digesting. Digest is available after reading of ARC which + // is too late for inclusion in WARC. + reader.setDigest(false); + // I don't want the close being logged -- least, not w/o log of + // an opening (and that'd be a little silly for simple script + // like this). Currently, it logs at level INFO so that close + // of files gets written to log files. Up the log level just + // for the close. + Logger l = Logger.getLogger(writer.getClass().getName()); + Level oldLevel = l.getLevel(); + try { + l.setLevel(Level.WARNING); + for (final Iterator i = reader.iterator(); i.hasNext();) { + WARCRecord r = (WARCRecord)i.next(); + if (!isARCType(r.getHeader().getMimetype())) { + continue; + } + if (r.getHeader().getContentBegin() <= 0) { + // Otherwise, because length include Header-Line and + // Named Fields, these will end up in the ARC unless there + // is a non-zero content begin. + continue; + } + String ip = (String)r.getHeader(). + getHeaderValue((WARCConstants.HEADER_KEY_IP)); + long length = r.getHeader().getLength(); + int offset = r.getHeader().getContentBegin(); + // This mimetype is not exactly what you'd expect to find in + // an ARC though technically its 'correct'. To get right one, + // need to parse the HTTP Headers. Thats messy. Not doing for + // now. + String mimetype = r.getHeader().getMimetype(); + // Clean out ISO time string '-', 'T', ':', and 'Z' characters. + String t = r.getHeader().getDate().replaceAll("[-T:Z]", ""); + long time = ArchiveUtils.getSecondsSinceEpoch(t).getTime(); + writer.write(r.getHeader().getUrl(), mimetype, ip, time, + (int)(length - offset), r); + } + } finally { + if (reader != null) { + reader.close(); + } + if (writer != null) { + try { + writer.close(); + } finally { + l.setLevel(oldLevel); + } + } + } + } + + protected boolean isARCType(final String mimetype) { + // Comparing mimetypes, especially WARC types can be problematic since + // they have whitespace. For now, ignore. + if (mimetype == null || mimetype.length() <= 0) { + return false; + } + String cleaned = mimetype.toLowerCase().trim(); + if (cleaned.equals(WARCConstants.HTTP_RESPONSE_MIMETYPE) || + cleaned.equals("text/dns")) { + return true; + } + return false; + } + + /** + * Command-line interface to Arc2Warc. + * + * @param args Command-line arguments. + * @throws ParseException Failed parse of the command line. + * @throws IOException + * @throws java.text.ParseException + */ + @SuppressWarnings("unchecked") +public static void main(String [] args) + throws ParseException, IOException, java.text.ParseException { + Options options = new Options(); + options.addOption(new Option("h","help", false, + "Prints this message and exits.")); + options.addOption(new Option("f","force", false, + "Force overwrite of target file.")); + options.addOption(new Option("p","prefix", true, + "Prefix to use on created ARC files, else uses default.")); + options.addOption(new Option("s","suffix", true, + "Suffix to use on created ARC files, else uses default.")); + PosixParser parser = new PosixParser(); + CommandLine cmdline = parser.parse(options, args, false); + List cmdlineArgs = cmdline.getArgList(); + Option [] cmdlineOptions = cmdline.getOptions(); + HelpFormatter formatter = new HelpFormatter(); + + // If no args, print help. + if (cmdlineArgs.size() < 0) { + usage(formatter, options, 0); + } + + // Now look at options passed. + boolean force = false; + String prefix = "WARC2ARC"; + String suffix = null; + for (int i = 0; i < cmdlineOptions.length; i++) { + switch(cmdlineOptions[i].getId()) { + case 'h': + usage(formatter, options, 0); + break; + + case 'f': + force = true; + break; + + case 'p': + prefix = cmdlineOptions[i].getValue(); + break; + + case 's': + suffix = cmdlineOptions[i].getValue(); + break; + + default: + throw new RuntimeException("Unexpected option: " + + + cmdlineOptions[i].getId()); + } + } + + // If no args, print help. + if (cmdlineArgs.size() != 2) { + usage(formatter, options, 0); + } + (new Warc2Arc()).transform(new File(cmdlineArgs.get(0).toString()), + new File(cmdlineArgs.get(1).toString()), prefix, suffix, force); + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/io/WriterPool.java b/commons/src/main/java/org/archive/io/WriterPool.java new file mode 100644 index 00000000..775f9a05 --- /dev/null +++ b/commons/src/main/java/org/archive/io/WriterPool.java @@ -0,0 +1,267 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io; + +import java.io.File; +import java.io.IOException; +import java.util.NoSuchElementException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.pool.BasePoolableObjectFactory; +import org.apache.commons.pool.impl.FairGenericObjectPool; +import org.apache.commons.pool.impl.GenericObjectPool; + +/** + * Pool of Writers. + * + * Abstract. Override and pass in the Constructor a factory that creates + * {@link WriterPoolMember} implementations. + * + * @author stack + */ +public abstract class WriterPool { + final Logger logger = Logger.getLogger(this.getClass().getName()); + + /** + * Used to generate unique filename sequences. + */ + final private AtomicInteger serialNo; + + /** + * Don't enforce a maximum number of idle instances in pool. + * To do so means GenericObjectPool will close files prematurely. + */ + protected static final int NO_MAX_IDLE = -1; + + /** + * Retry getting a file on fail the below arbitrary amount of times. + * This facility is not configurable. If we fail this many times + * getting a file, something is seriously wrong. + */ + private final int arbitraryRetryMax = 10; + + /** + * Default maximum active number of files in the pool. + */ + public static final int DEFAULT_MAX_ACTIVE = 5; + + /** + * Maximum time to wait on a free file.. + */ + public static final int DEFAULT_MAXIMUM_WAIT = 1000 * 60 * 5; + + /** + * Pool instance. + */ + private GenericObjectPool pool = null; + + /** + * File settings. + * Keep in data structure rather than as individual values. + */ + private final WriterPoolSettings settings; + + /** + * Shutdown default constructor. + */ + @SuppressWarnings("unused") + private WriterPool() { + this(null, null, null, -1, -1); + } + + /** + * Constructor + * @param serial Used to generate unique filename sequences + * @param factory Factory that knows how to make a {@link WriterPoolMember}. + * @param settings Settings for this pool. + * @param poolMaximumActive + * @param poolMaximumWait + */ + public WriterPool(final AtomicInteger serial, + final BasePoolableObjectFactory factory, + final WriterPoolSettings settings, + final int poolMaximumActive, final int poolMaximumWait) { + logger.info("Initial configuration:" + + " prefix=" + settings.getPrefix() + + ", suffix=" + settings.getSuffix() + + ", compress=" + settings.isCompressed() + + ", maxSize=" + settings.getMaxSize() + + ", maxActive=" + poolMaximumActive + + ", maxWait=" + poolMaximumWait); + this.settings = settings; + this.pool = new FairGenericObjectPool(factory, poolMaximumActive, + GenericObjectPool.WHEN_EXHAUSTED_BLOCK, poolMaximumWait, + NO_MAX_IDLE); + this.serialNo = serial; + } + + /** + * Check out a {@link WriterPoolMember}. + * + * This method must be answered by a call to + * {@link #returnFile(WriterPoolMember)} else pool starts leaking. + * + * @return Writer checked out of a pool of files. + * @throws IOException Problem getting Writer from pool (Converted + * from Exception to IOException so this pool can live as a good citizen + * down in depths of ARCSocketFactory). + * @throws NoSuchElementException If we time out waiting on a pool member. + */ + public WriterPoolMember borrowFile() + throws IOException { + WriterPoolMember f = null; + for (int i = 0; f == null; i++) { + long waitStart = System.currentTimeMillis(); + try { + f = (WriterPoolMember)this.pool.borrowObject(); + if (logger.getLevel() == Level.FINE) { + logger.fine("Borrowed " + f + " (Pool State: " + + getPoolState(waitStart) + ")."); + } + } catch (NoSuchElementException e) { + // Let this exception out. Unit test at least depends on it. + // Log current state of the pool. + logger.warning(e.getMessage() + ": Retry #" + i + " of " + + " max of " + arbitraryRetryMax + + ": NSEE Pool State: " + getPoolState(waitStart)); + if (i >= arbitraryRetryMax) { + logger.log(Level.SEVERE, + "maximum retries exceeded; rethrowing",e); + throw e; + } + } catch (Exception e) { + // Convert. + logger.log(Level.SEVERE,"E Pool State: " + + getPoolState(waitStart), e); + throw new IOException("Failed getting writer from pool: " + + e.getMessage()); + } + } + return f; + } + + /** + * @param writer Writer to return to the pool. + * @throws IOException Problem returning File to pool. + */ + public void returnFile(WriterPoolMember writer) + throws IOException { + try { + if (logger.getLevel() == Level.FINE) { + logger.fine("Returned " + writer); + } + this.pool.returnObject(writer); + } + catch(Exception e) + { + throw new IOException("Failed restoring writer to pool: " + + e.getMessage()); + } + } + + public void invalidateFile(WriterPoolMember f) + throws IOException { + try { + this.pool.invalidateObject(f); + } catch (Exception e) { + // Convert exception. + throw new IOException(e.getMessage()); + } + // It'll have been closed. Rename with an '.invalid' suffix so it + // gets attention. + File file = f.getFile(); + file.renameTo(new File(file.getAbsoluteFile() + + WriterPoolMember.INVALID_SUFFIX)); + } + + /** + * @return Number of {@link WriterPoolMember}s checked out of pool. + * @throws java.lang.UnsupportedOperationException + */ + public int getNumActive() + throws UnsupportedOperationException { + return this.pool.getNumActive(); + } + + /** + * @return Number of {@link WriterPoolMember} instances still in the pool. + * @throws java.lang.UnsupportedOperationException + */ + public int getNumIdle() + throws UnsupportedOperationException { + return this.pool.getNumIdle(); + } + + /** + * Close all {@link WriterPoolMember}s in pool. + */ + public void close() { + this.pool.clear(); + } + + /** + * @return Returns settings. + */ + public WriterPoolSettings getSettings() { + return this.settings; + } + + /** + * @return State of the pool string + */ + protected String getPoolState() { + return getPoolState(-1); + } + + /** + * @param startTime If we are passed a start time, we'll add difference + * between it and now to end of string. Pass -1 if don't want this + * added to end of state string. + * @return State of the pool string + */ + protected String getPoolState(long startTime) { + StringBuffer buffer = new StringBuffer("Active "); + buffer.append(getNumActive()); + buffer.append(" of max "); + buffer.append(this.pool.getMaxActive()); + buffer.append(", idle "); + buffer.append(this.pool.getNumIdle()); + if (startTime != -1) { + buffer.append(", time "); + buffer.append(System.currentTimeMillis() - startTime); + buffer.append("ms of max "); + buffer.append(this.pool.getMaxWait()); + buffer.append("ms"); + } + return buffer.toString(); + } + + /** + * Returns the atomic integer used to generate serial numbers + * for files. + * + * @return the serial number generator + */ + public AtomicInteger getSerialNo() { + return serialNo; + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/io/WriterPoolMember.java b/commons/src/main/java/org/archive/io/WriterPoolMember.java new file mode 100644 index 00000000..6d4ca775 --- /dev/null +++ b/commons/src/main/java/org/archive/io/WriterPoolMember.java @@ -0,0 +1,538 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io; + +import it.unimi.dsi.fastutil.io.FastBufferedOutputStream; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.text.DecimalFormat; +import java.text.NumberFormat; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Logger; +import java.util.zip.GZIPOutputStream; + +import org.archive.util.ArchiveUtils; +import org.archive.util.IoUtils; +import org.archive.util.TimestampSerialno; + + +/** + * Member of {@link WriterPool}. + * Implements rotating off files, file naming with some guarantee of + * uniqueness, and position in file. Subclass to pick up functionality for a + * particular Writer type. + * @author stack + * @version $Date$ $Revision$ + */ +public abstract class WriterPoolMember implements ArchiveFileConstants { + private final Logger logger = Logger.getLogger(this.getClass().getName()); + + public static final String UTF8 = "UTF-8"; + + /** + * Default file prefix. + * + * Stands for Internet Archive Heritrix. + */ + public static final String DEFAULT_PREFIX = "IAH"; + + /** + * Value to interpolate with actual hostname. + */ + public static final String HOSTNAME_VARIABLE = "${HOSTNAME}"; + + /** + * Default for file suffix. + */ + public static final String DEFAULT_SUFFIX = HOSTNAME_VARIABLE; + + /** + * Reference to file we're currently writing. + */ + private File f = null; + + /** + * Output stream for file. + */ + private OutputStream out = null; + + /** + * File output stream. + * This is needed so can get at channel to find current position in file. + */ + private FileOutputStream fos; + + private final boolean compressed; + private List writeDirs = null; + private String prefix = DEFAULT_PREFIX; + private String suffix = DEFAULT_SUFFIX; + private final long maxSize; + private final String extension; + + /** + * Creation date for the current file. + * Set by {@link #createFile()}. + */ + private String createTimestamp = "UNSET!!!"; + + /** + * A running sequence used making unique file names. + */ + final private AtomicInteger serialNo; + + /** + * Directories round-robin index. + */ + private static int roundRobinIndex = 0; + + /** + * NumberFormat instance for formatting serial number. + * + * Pads serial number with zeros. + */ + private static NumberFormat serialNoFormatter = new DecimalFormat("00000"); + + + /** + * Buffer to reuse writing streams. + */ + private final byte [] scratchbuffer = new byte[4 * 1024]; + + + /** + * Constructor. + * Takes a stream. Use with caution. There is no upperbound check on size. + * Will just keep writing. + * + * @param serialNo used to create unique filename sequences + * @param out Where to write. + * @param file File the out is connected to. + * @param cmprs Compress the content written. + * @param a14DigitDate If null, we'll write current time. + * @throws IOException + */ + protected WriterPoolMember(AtomicInteger serialNo, + final OutputStream out, final File file, + final boolean cmprs, String a14DigitDate) + throws IOException { + this(serialNo, null, null, cmprs, -1, null); + this.out = out; + this.f = file; + } + + /** + * Constructor. + * + * @param serialNo used to create unique filename sequences + * @param dirs Where to drop files. + * @param prefix File prefix to use. + * @param cmprs Compress the records written. + * @param maxSize Maximum size for ARC files written. + * @param extension Extension to give file. + */ + public WriterPoolMember(AtomicInteger serialNo, + final List dirs, final String prefix, + final boolean cmprs, final long maxSize, final String extension) { + this(serialNo, dirs, prefix, "", cmprs, maxSize, extension); + } + + /** + * Constructor. + * + * @param serialNo used to create unique filename sequences + * @param dirs Where to drop files. + * @param prefix File prefix to use. + * @param cmprs Compress the records written. + * @param maxSize Maximum size for ARC files written. + * @param suffix File tail to use. If null, unused. + * @param extension Extension to give file. + */ + public WriterPoolMember(AtomicInteger serialNo, + final List dirs, final String prefix, + final String suffix, final boolean cmprs, + final long maxSize, final String extension) { + this.suffix = suffix; + this.prefix = prefix; + this.maxSize = maxSize; + this.writeDirs = dirs; + this.compressed = cmprs; + this.extension = extension; + this.serialNo = serialNo; + } + + /** + * Call this method just before/after any significant write. + * + * Call at the end of the writing of a record or just before we start + * writing a new record. Will close current file and open a new file + * if file size has passed out maxSize. + * + *

Creates and opens a file if none already open. One use of this method + * then is after construction, call this method to add the metadata, then + * call {@link #getPosition()} to find offset of first record. + * + * @exception IOException + */ + public void checkSize() throws IOException { + if (this.out == null || + (this.maxSize != -1 && (this.f.length() > this.maxSize))) { + createFile(); + } + } + + /** + * Create a new file. + * Rotates off the current Writer and creates a new in its place + * to take subsequent writes. Usually called from {@link #checkSize()}. + * @return Name of file created. + * @throws IOException + */ + protected String createFile() throws IOException { + TimestampSerialno tsn = getTimestampSerialNo(); + String name = this.prefix + '-' + getUniqueBasename(tsn) + + ((this.suffix == null || this.suffix.length() <= 0)? + "": "-" + this.suffix) + '.' + this.extension + + ((this.compressed)? '.' + COMPRESSED_FILE_EXTENSION: "") + + OCCUPIED_SUFFIX; + this.createTimestamp = tsn.getTimestamp(); + File dir = getNextDirectory(this.writeDirs); + return createFile(new File(dir, name)); + } + + protected String createFile(final File file) throws IOException { + close(); + this.f = file; + this.fos = new FileOutputStream(this.f); + this.out = new FastBufferedOutputStream(this.fos); + logger.info("Opened " + this.f.getAbsolutePath()); + return this.f.getName(); + } + + /** + * @param dirs List of File objects that point at directories. + * @return Find next directory to write an arc too. If more + * than one, it tries to round-robin through each in turn. + * @throws IOException + */ + protected File getNextDirectory(List dirs) + throws IOException { + if (WriterPoolMember.roundRobinIndex >= dirs.size()) { + WriterPoolMember.roundRobinIndex = 0; + } + File d = null; + try { + d = checkWriteable((File)dirs. + get(WriterPoolMember.roundRobinIndex)); + } catch (IndexOutOfBoundsException e) { + // Dirs list might be altered underneath us. + // If so, we get this exception -- just keep on going. + } + if (d == null && dirs.size() > 1) { + for (Iterator i = dirs.iterator(); d == null && i.hasNext();) { + d = checkWriteable((File)i.next()); + } + } else { + WriterPoolMember.roundRobinIndex++; + } + if (d == null) { + throw new IOException("Directories unusable."); + } + return d; + } + + protected File checkWriteable(File d) { + if (d == null) { + return d; + } + + try { + IoUtils.ensureWriteableDirectory(d); + } catch(IOException e) { + logger.warning("Directory " + d.getPath() + " is not" + + " writeable or cannot be created: " + e.getMessage()); + d = null; + } + return d; + } + + protected synchronized TimestampSerialno getTimestampSerialNo() { + return getTimestampSerialNo(null); + } + + /** + * Do static synchronization around getting of counter and timestamp so + * no chance of a thread getting in between the getting of timestamp and + * allocation of serial number throwing the two out of alignment. + * + * @param timestamp If non-null, use passed timestamp (must be 14 digit + * ARC format), else if null, timestamp with now. + * @return Instance of data structure that has timestamp and serial no. + */ + protected synchronized TimestampSerialno + getTimestampSerialNo(final String timestamp) { + return new TimestampSerialno((timestamp != null)? + timestamp: ArchiveUtils.get14DigitDate(), + serialNo.getAndIncrement()); + } + + /** + * Return a unique basename. + * + * Name is timestamp + an every increasing sequence number. + * + * @param tsn Structure with timestamp and serial number. + * + * @return Unique basename. + */ + private String getUniqueBasename(TimestampSerialno tsn) { + return tsn.getTimestamp() + "-" + + WriterPoolMember.serialNoFormatter.format(tsn.getSerialNumber()); + } + + + /** + * Get the file name + * + * @return the filename, as if uncompressed + */ + protected String getBaseFilename() { + String name = this.f.getName(); + if (this.compressed && name.endsWith(DOT_COMPRESSED_FILE_EXTENSION)) { + return name.substring(0,name.length() - 3); + } else if(this.compressed && + name.endsWith(DOT_COMPRESSED_FILE_EXTENSION + + OCCUPIED_SUFFIX)) { + return name.substring(0, name.length() - + (3 + OCCUPIED_SUFFIX.length())); + } else { + return name; + } + } + + /** + * Get this file. + * + * Used by junit test to test for creation and when {@link WriterPool} wants + * to invalidate a file. + * + * @return The current file. + */ + public File getFile() { + return this.f; + } + + /** + * Post write tasks. + * + * Has side effects. Will open new file if we're at the upperbound. + * If we're writing compressed files, it will wrap output stream with a + * GZIP writer with side effect that GZIP header is written out on the + * stream. + * + * @exception IOException + */ + protected void preWriteRecordTasks() + throws IOException { + checkSize(); + if (this.compressed) { + // Wrap stream in GZIP Writer. + // The below construction immediately writes the GZIP 'default' + // header out on the underlying stream. + this.out = new CompressedStream(this.out); + } + } + + /** + * Post file write tasks. + * If compressed, finishes up compression and flushes stream so any + * subsequent checks get good reading. + * + * @exception IOException + */ + protected void postWriteRecordTasks() + throws IOException { + if (this.compressed) { + CompressedStream o = (CompressedStream)this.out; + o.finish(); + o.flush(); + o.end(); + this.out = o.getWrappedStream(); + } + } + + /** + * Postion in current physical file. + * Used making accounting of bytes written. + * @return Position in underlying file. Call before or after writing + * records *only* to be safe. + * @throws IOException + */ + public long getPosition() throws IOException { + long position = 0; + if (this.out != null) { + this.out.flush(); + } + if (this.fos != null) { + // Call flush on underlying file though probably not needed assuming + // above this.out.flush called through to this.fos. + this.fos.flush(); + position = this.fos.getChannel().position(); + } + return position; + } + + public boolean isCompressed() { + return compressed; + } + + protected void write(final byte [] b) throws IOException { + this.out.write(b); + } + + protected void flush() throws IOException { + this.out.flush(); + } + + protected void write(byte[] b, int off, int len) throws IOException { + this.out.write(b, off, len); + } + + protected void write(int b) throws IOException { + this.out.write(b); + } + + /** + * @deprecated Use {@link #copyFrom(InputStream,long,boolean)} instead + */ + protected void readFullyFrom(final InputStream is, final long recordLength, + final byte [] b) + throws IOException { + copyFrom(is, recordLength, true); + } + + /** + * @deprecated Use {@link #copyFrom(InputStream,long,boolean)} instead + */ + protected void readToLimitFrom(final InputStream is, final long limit, + final byte [] b) + throws IOException { + copyFrom(is, limit, true); + } + + /** + * Copy bytes from the provided InputStream to the target file/stream being + * written. + * + * @param is + * InputStream to copy bytes from + * @param recordLength + * expected number of bytes to copy + * @param enforceLength + * whether to throw an exception if too many/too few bytes are + * available from stream + * @throws IOException + */ + protected void copyFrom(final InputStream is, final long recordLength, + boolean enforceLength) throws IOException { + int read = scratchbuffer.length; + long tot = 0; + while ((tot < recordLength) + && (read = is.read(scratchbuffer)) != -1) { + int write = read; + // never write more than enforced length + write = (int) Math.min(write, recordLength - tot); + tot += read; + write(scratchbuffer, 0, write); + } + if (enforceLength && tot != recordLength) { + // throw exception if desired for read vs. declared mismatches + throw new IOException("Read " + tot + " but expected " + + recordLength); + } + } + + public void close() throws IOException { + if (this.out == null) { + return; + } + this.out.close(); + this.out = null; + this.fos = null; + if (this.f != null && this.f.exists()) { + String path = this.f.getAbsolutePath(); + if (path.endsWith(OCCUPIED_SUFFIX)) { + File f = new File(path.substring(0, + path.length() - OCCUPIED_SUFFIX.length())); + if (!this.f.renameTo(f)) { + logger.warning("Failed rename of " + path); + } + this.f = f; + } + + logger.info("Closed " + this.f.getAbsolutePath() + + ", size " + this.f.length()); + } + } + + protected OutputStream getOutputStream() { + return this.out; + } + + protected String getCreateTimestamp() { + return createTimestamp; + } + + + /** + * An override so we get access to underlying output stream. + * and offer an end() that does not accompany closing underlying + * stream. + * @author stack + */ + private class CompressedStream extends GZIPOutputStream { + public CompressedStream(OutputStream out) + throws IOException { + super(out); + } + + /** + * @return Reference to stream being compressed. + */ + OutputStream getWrappedStream() { + return this.out; + } + + /** + * Release the deflater's native process resources, + * which otherwise would not occur until either + * finalization or DeflaterOutputStream.close() + * (which would also close underlying stream). + */ + public void end() { + def.end(); + } + } +} diff --git a/commons/src/main/java/org/archive/io/WriterPoolSettings.java b/commons/src/main/java/org/archive/io/WriterPoolSettings.java new file mode 100644 index 00000000..c9812175 --- /dev/null +++ b/commons/src/main/java/org/archive/io/WriterPoolSettings.java @@ -0,0 +1,37 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + +import java.io.File; +import java.util.List; + +/** + * Settings object for a {@link WriterPool}. + * Used creating {@link WriterPoolMember}s. + * @author stack + * @version $Date$, $Revision$ + */ +public interface WriterPoolSettings { + public long getMaxSize(); + public String getPrefix(); + public String getSuffix(); + public List getOutputDirs(); + public boolean isCompressed(); + public List getMetadata(); +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/io/arc/ARC2WCDX.java b/commons/src/main/java/org/archive/io/arc/ARC2WCDX.java new file mode 100644 index 00000000..19010131 --- /dev/null +++ b/commons/src/main/java/org/archive/io/arc/ARC2WCDX.java @@ -0,0 +1,243 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io.arc; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.util.Date; +import java.util.Iterator; +import java.util.zip.GZIPOutputStream; + +import org.apache.commons.httpclient.Header; +import org.apache.commons.httpclient.HeaderGroup; +import org.apache.commons.httpclient.util.DateParseException; +import org.apache.commons.httpclient.util.DateUtil; +import org.archive.io.ArchiveRecord; +import org.archive.util.ArchiveUtils; +import org.archive.util.SURT; + +/** + * Create a 'Wide' CDX from an ARC. Takes one argument, the path to the ARC. + * Writes .wcdx.gz in same directory. + * + * @author gojomo + */ +public class ARC2WCDX { + final public static String WCDX_VERSION="0.1"; + + public static void main(String[] args) throws IOException { + String arcFilename = args[0]; + createWcdx(arcFilename); + } + + public static Object[] createWcdx(String arcFilename) throws IOException { + ARCReader reader = ARCReaderFactory.get(arcFilename); + Object[] retVal = createWcdx(reader); + reader.close(); + return retVal; + } + + public static Object[] createWcdx(ARCReader reader) { + reader.setDigest(true); + + String wcdxPath = reader.getReaderIdentifier().replaceAll("\\.arc(\\.gz)?$",".wcdx.gz"); + File wcdxFile = new File(wcdxPath+".open"); + PrintStream writer = null; + long count = 0; + try { + writer = new PrintStream(new GZIPOutputStream(new FileOutputStream(wcdxFile))); + + // write header: legend + timestamp + StringBuilder legend = new StringBuilder(); + appendField(legend,"CDX"); + appendField(legend,"surt-uri"); + appendField(legend,"b"); // ARC timestamp + appendField(legend,"http-date"); + appendField(legend,"s"); // status code + appendField(legend,"m"); // media type + appendField(legend,"sha1"); // content sha1 + appendField(legend,"g"); // ARC name + appendField(legend,"V"); // start offset + appendField(legend,"end-offset"); // TODO: implement + appendField(legend,"n"); // ARC record length TODO: verify + appendField(legend,"http-content-length"); + appendField(legend,"http-last-modified"); + appendField(legend,"http-expires"); + appendField(legend,"http-etag"); + appendField(legend,"http-location"); + appendField(legend,"e"); // IP + appendField(legend,"a"); // original URL + // WCDX version+creation time: crude version control + appendField(legend,WCDX_VERSION+"@"+ArchiveUtils.get14DigitDate()); + writer.println(legend.toString()); + + Iterator iter = reader.iterator(); + count = 0; + while(iter.hasNext()) { + ARCRecord record = (ARCRecord) iter.next(); + record.close(); + ARCRecordMetaData h = (ARCRecordMetaData) record.getHeader(); + Header[] httpHeaders = record.getHttpHeaders(); + if(httpHeaders==null) { + httpHeaders = new Header[0]; + } + HeaderGroup hg = new HeaderGroup(); + hg.setHeaders(httpHeaders); + StringBuilder builder = new StringBuilder(); + + // SURT-form URI + appendField(builder,SURT.fromURI(h.getUrl())); + // record timestamp ('b') + appendField(builder,h.getDate()); + // http header date + appendTimeField(builder,hg.getFirstHeader("Date")); + // response code ('s') + appendField(builder,h.getStatusCode()); + // media type ('m') + appendField(builder,h.getMimetype()); + // content checksum (like 'c', but here Base32 SHA1) + appendField(builder,record.getDigestStr()); + // arc name ('g') + appendField(builder,reader.getFileName()); + // compressed start offset ('V') + appendField(builder,h.getOffset()); + + // compressed end offset (?) +// appendField(builder, +// reader.getInputStream() instanceof RepositionableStream +// ? ((GzippedInputStream)reader.getInputStream()).vPosition() +// : "-"); + // TODO; leave unavail for now + appendField(builder, "-"); + + // uncompressed (declared in ARC headerline) record length + appendField(builder,h.getLength()); + // http header content-length + appendField(builder,hg.getFirstHeader("Content-Length")); + + // http header mod-date + appendTimeField(builder,hg.getFirstHeader("Last-Modified")); + // http header expires + appendTimeField(builder,hg.getFirstHeader("Expires")); + + // http header etag + appendField(builder,hg.getFirstHeader("ETag")); + // http header redirect ('Location' header?) + appendField(builder,hg.getFirstHeader("Location")); + // ip ('e') + appendField(builder,h.getIp()); + // original URI + appendField(builder,h.getUrl()); + // TODO MAYBE - a title from inside content? + + writer.println(builder.toString()); + count++; + } + wcdxFile.renameTo(new File(wcdxPath)); + } catch (IOException e) { + // soldier on: but leave '.open' wcdx file as indicator of error + if(!wcdxFile.exists()) { + try { + wcdxFile.createNewFile(); + } catch (IOException e1) { + // TODO Auto-generated catch block + throw new RuntimeException(e1); + } + } + } catch (RuntimeException e) { + // soldier on: but leave '.open' wcdx file as indicator of error + if(!wcdxFile.exists()) { + try { + wcdxFile.createNewFile(); + } catch (IOException e1) { + // TODO Auto-generated catch block + throw new RuntimeException(e1); + } + } + } finally { + if(writer!=null) { + writer.close(); + } + } + + return new Object[] {wcdxPath, count}; + } + + protected static void appendField(StringBuilder builder, Object obj) { + if(builder.length()>0) { + // prepend with delimiter + builder.append(' '); + } + if(obj instanceof Header) { + obj = ((Header)obj).getValue().trim(); + } + + builder.append((obj==null||obj.toString().length()==0)?"-":obj); + } + + protected static void appendTimeField(StringBuilder builder, Object obj) { + if(builder.length()>0) { + // prepend with delimiter + builder.append(' '); + } + if(obj==null) { + builder.append("-"); + return; + } + if(obj instanceof Header) { + String s = ((Header)obj).getValue().trim(); + try { + Date date = DateUtil.parseDate(s); + String d = ArchiveUtils.get14DigitDate(date); + if(d.startsWith("209")) { + d = "199"+d.substring(3); + } + obj = d; + } catch (DateParseException e) { + builder.append('e'); + return; + } + + } + builder.append(obj); + } +} + +//'wide' CDX +//a original url +//b timestamp +//s resp code +//m type +//? content md5 (full 'k'? 'c'? +//g arc name +//V compressed start offset +//? compressed length +//n? uncompressed length +//? mod date +//? expires +//? server 'date' hdr +//? etag +//r redirect ('Location'?) +//e ip +//MAYBE: +//? TITLE from HTML or other format? + + diff --git a/commons/src/main/java/org/archive/io/arc/ARCConstants.java b/commons/src/main/java/org/archive/io/arc/ARCConstants.java new file mode 100644 index 00000000..c587aa1b --- /dev/null +++ b/commons/src/main/java/org/archive/io/arc/ARCConstants.java @@ -0,0 +1,227 @@ +/* + * ARCConstants + * + * $Id$ + * + * Created on Dec 30, 2003. + * + * Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.io.arc; + +import java.util.Arrays; +import java.util.List; +import java.util.zip.Deflater; +import java.util.zip.GZIPInputStream; + +import org.archive.io.ArchiveFileConstants; +import org.archive.io.GzipHeader; + +/** + * Constants used by ARC files and in ARC file processing. + * + * @author stack + */ +public interface ARCConstants extends ArchiveFileConstants { + /** + * Default maximum ARC file size. + */ + public static final long DEFAULT_MAX_ARC_FILE_SIZE = 100000000; + + /** + * Maximum length for a metadata line. + */ + public static final int MAX_METADATA_LINE_LENGTH = (4 * 1024); + + /** + * ARC file extention. + */ + public static final String ARC_FILE_EXTENSION = "arc"; + + /** + * Dot ARC file extension. + */ + public static final String DOT_ARC_FILE_EXTENSION = + "." + ARC_FILE_EXTENSION; + + public static final String DOT_COMPRESSED_FILE_EXTENSION = + ArchiveFileConstants.DOT_COMPRESSED_FILE_EXTENSION; + + /** + * Compressed arc file extension. + */ + public static final String COMPRESSED_ARC_FILE_EXTENSION = + ARC_FILE_EXTENSION + DOT_COMPRESSED_FILE_EXTENSION; + + /** + * Compressed dot arc file extension. + */ + public static final String DOT_COMPRESSED_ARC_FILE_EXTENSION = + DOT_ARC_FILE_EXTENSION + DOT_COMPRESSED_FILE_EXTENSION; + + /** + * Encoding to use getting bytes from strings. + * + * Specify an encoding rather than leave it to chance: i.e whatever the + * JVMs encoding. Use an encoding that gets the stream as bytes, not chars. + */ + public static final String DEFAULT_ENCODING = "ISO-8859-1"; + + /** + * ARC file line seperator character. + * + * This is what the alexa c-code looks for delimiting lines. + */ + public static final char LINE_SEPARATOR = '\n'; + + /** + * ARC header field seperator character. + */ + public static final char HEADER_FIELD_SEPARATOR = ' '; + + /** + * ARC file *MAGIC NUMBER*. + * + * Every ARC file must begin w/ this. + */ + public static final String ARC_MAGIC_NUMBER = "filedesc://"; + + /** + * The FLG.FEXTRA field that is added to ARC files. (See RFC1952 to + * understand FLG.FEXTRA). + */ + public static final byte[] ARC_GZIP_EXTRA_FIELD = { 8, 0, 'L', 'X', 4, 0, + 0, 0, 0, 0 }; + + /** + * Key for the ARC Header IP field. + * + * Lowercased. + */ + public static final String IP_HEADER_FIELD_KEY = "ip-address"; + + /** + * Key for the ARC Header Result Code field. + * + * Lowercased. + */ + public static final String CODE_HEADER_FIELD_KEY = "result-code"; + + /** + * Key for the ARC Header Checksum field. + * + * Lowercased. + */ + public static final String CHECKSUM_HEADER_FIELD_KEY = "checksum"; + + /** + * Key for the ARC Header Location field. + * + * Lowercased. + */ + public static final String LOCATION_HEADER_FIELD_KEY = "location"; + + /** + * Key for the ARC Header Offset field. + * + * Lowercased. + */ + public static final String OFFSET_HEADER_FIELD_KEY = "offset"; + + /** + * Key for the ARC Header filename field. + * + * Lowercased. + */ + public static final String FILENAME_HEADER_FIELD_KEY = "filename"; + + /** + * Key for statuscode field. + */ + public static final String STATUSCODE_FIELD_KEY = "statuscode"; + + /** + * Key for offset field. + */ + public static final String OFFSET_FIELD_KEY = OFFSET_HEADER_FIELD_KEY; + + /** + * Key for filename field. + */ + public static final String FILENAME_FIELD_KEY = FILENAME_HEADER_FIELD_KEY; + + /** + * Key for checksum field. + */ + public static final String CHECKSUM_FIELD_KEY = CHECKSUM_HEADER_FIELD_KEY; + + /** + * Tokenized field prefix. + * + * Use this prefix for tokenized fields when naming fields in + * an index. + */ + public static final String TOKENIZED_PREFIX = "tokenized_"; + + /** + * Assumed maximum size of a record meta header line. + * + * This 100k which seems massive but its the same as the LINE_LENGTH from + * alexa/include/a_arcio.h: + * 

+     * #define LINE_LENGTH     (100*1024)
+     *
+ */ + public static final int MAX_HEADER_LINE_LENGTH = 1024 * 100; + + /** + * Version 1 required metadata fields. + */ + public static List REQUIRED_VERSION_1_HEADER_FIELDS = Arrays + .asList(new String[] { URL_FIELD_KEY, IP_HEADER_FIELD_KEY, + DATE_FIELD_KEY, MIMETYPE_FIELD_KEY, + LENGTH_FIELD_KEY, VERSION_FIELD_KEY, + ABSOLUTE_OFFSET_KEY }); + + /** + * Minimum possible record length. + * + * This is a rough calc. When the header is data it will occupy less space. + */ + public static int MINIMUM_RECORD_LENGTH = 1 + "://".length() + 1 + + ARC_FILE_EXTENSION.length() + " ".length() + +1 + " ".length() + + 1 + " ".length() + 1 + "/".length() + 1 + " ".length() + 1; + + /** + * Start of a GZIP header that uses default deflater. + */ + public static final byte[] GZIP_HEADER_BEGIN = { + (byte) GZIPInputStream.GZIP_MAGIC, // Magic number (short) + (byte) (GZIPInputStream.GZIP_MAGIC >> 8), // Magic number (short) + Deflater.DEFLATED // Compression method (CM) + }; + + /** + * Length of minimual 'default GZIP header. + * + * See RFC1952 for explaination of value of 10. + */ + public static final int DEFAULT_GZIP_HEADER_LENGTH = + GzipHeader.MINIMAL_GZIP_HEADER_LENGTH; +} diff --git a/commons/src/main/java/org/archive/io/arc/ARCLocation.java b/commons/src/main/java/org/archive/io/arc/ARCLocation.java new file mode 100644 index 00000000..227d9e89 --- /dev/null +++ b/commons/src/main/java/org/archive/io/arc/ARCLocation.java @@ -0,0 +1,43 @@ +/* ARCLocation + * + * $Id$ + * + * Created on April 27, 2005. + * + * Copyright (C) 2005 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or any + * later version. + * + * The archive-access tools are distributed in the hope that they will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser + * Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License along with + * the archive-access tools; if not, write to the Free Software Foundation, + * Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.io.arc; + +/** + * Datastructure to hold ARC record location. + * Used by wayback machine. + * @author stack + */ +public interface ARCLocation { + /** + * @return Returns the ARC filename. Can be full path to ARC, URL to an + * ARC or just the portion of an ARC name that is unique to a collection. + */ + public String getName(); + + /** + * @return Returns the offset into the ARC. + */ + public long getOffset(); +} diff --git a/commons/src/main/java/org/archive/io/arc/ARCReader.java b/commons/src/main/java/org/archive/io/arc/ARCReader.java new file mode 100644 index 00000000..7b88f7bb --- /dev/null +++ b/commons/src/main/java/org/archive/io/arc/ARCReader.java @@ -0,0 +1,551 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io.arc; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Logger; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; +import org.archive.io.ArchiveReader; +import org.archive.io.ArchiveRecord; +import org.archive.io.ArchiveRecordHeader; +import org.archive.io.RecoverableIOException; +import org.archive.io.WriterPoolMember; +import org.archive.util.ArchiveUtils; + + +/** + * Get an iterator on an ARC file or get a record by absolute position. + * + * ARC files are described here: + * Arc + * File Format. + * + *

This class knows how to parse an ARC file. Pass it a file path + * or an URL to an ARC. It can parse ARC Version 1 and 2. + * + *

Iterator returns ARCRecord + * though {@link Iterator#next()} is returning + * java.lang.Object. Cast the return. + * + *

Profiling java.io vs. memory-mapped ByteBufferInputStream shows the + * latter slightly slower -- but not by much. TODO: Test more. Just + * change {@link #getInputStream(File, long)}. + * + * @author stack + * @version $Date$ $Revision$ + */ +public abstract class ARCReader extends ArchiveReader +implements ARCConstants { + Logger logger = Logger.getLogger(ARCReader.class.getName()); + + /** + * Set to true if we are aligned on first record of Archive file. + * We used depend on offset. If offset was zero, then we were + * aligned on first record. This is no longer necessarily the case when + * Reader is created at an offset into an Archive file: The offset is zero + * but its relative to where we started reading. + */ + private boolean alignedOnFirstRecord = true; + + private boolean parseHttpHeaders = true; + + ARCReader() { + super(); + } + + /** + * Skip over any trailing new lines at end of the record so we're lined up + * ready to read the next. + * @param record + * @throws IOException + */ + protected void gotoEOR(ArchiveRecord record) throws IOException { + if (getIn().available() <= 0) { + return; + } + + // Remove any trailing LINE_SEPARATOR + int c = -1; + while (getIn().available() > 0) { + if (getIn().markSupported()) { + getIn().mark(1); + } + c = getIn().read(); + if (c != -1) { + if (c == LINE_SEPARATOR) { + continue; + } + if (getIn().markSupported()) { + // We've overread. We're probably in next record. There is + // no way of telling for sure. It may be dross at end of + // current record. Backup. + getIn().reset(); + break; + } + ArchiveRecordHeader h = (getCurrentRecord() != null)? + record.getHeader(): null; + throw new IOException("Read " + (char)c + + " when only " + LINE_SEPARATOR + " expected. " + + getReaderIdentifier() + ((h != null)? + h.getHeaderFields().toString(): "")); + } + } + } + + /** + * Create new arc record. + * + * Encapsulate housekeeping that has to do w/ creating a new record. + * + *

Call this method at end of constructor to read in the + * arcfile header. Will be problems reading subsequent arc records + * if you don't since arcfile header has the list of metadata fields for + * all records that follow. + * + *

When parsing through ARCs writing out CDX info, we spend about + * 38% of CPU in here -- about 30% of which is in getTokenizedHeaderLine + * -- of which 16% is reading. + * + * @param is InputStream to use. + * @param offset Absolute offset into arc file. + * @return An arc record. + * @throws IOException + */ + protected ARCRecord createArchiveRecord(InputStream is, long offset) + throws IOException { + try { + String version = super.getVersion(); + ARCRecord record = new ARCRecord(is, getReaderIdentifier(), offset, + isDigest(), isStrict(), isParseHttpHeaders(), + isAlignedOnFirstRecord(), version); + if (version != null && super.getVersion() == null) + super.setVersion(version); + currentRecord(record); + } catch (IOException e) { + if (e instanceof RecoverableIOException) { + // Don't mess with RecoverableIOExceptions. Let them out. + throw e; + } + IOException newE = new IOException(e.getMessage() + " (Offset " + + offset + ")."); + newE.setStackTrace(e.getStackTrace()); + throw newE; + } + return (ARCRecord)getCurrentRecord(); + } + + /** + * Returns version of this ARC file. Usually read from first record of ARC. + * If we're reading without having first read the first record -- e.g. + * random access into middle of an ARC -- then version will not have been + * set. For now, we return a default, version 1.1. Later, if more than + * just one version of ARC, we could look at such as the meta line to see + * what version of ARC this is. + * @return Version of this ARC file. + */ + public String getVersion() { + return (super.getVersion() == null)? "1.1": super.getVersion(); + } + + protected boolean isAlignedOnFirstRecord() { + return alignedOnFirstRecord; + } + + protected void setAlignedOnFirstRecord(boolean alignedOnFirstRecord) { + this.alignedOnFirstRecord = alignedOnFirstRecord; + } + + /** + * @return Returns the parseHttpHeaders. + */ + public boolean isParseHttpHeaders() { + return this.parseHttpHeaders; + } + + /** + * @param parse The parseHttpHeaders to set. + */ + public void setParseHttpHeaders(boolean parse) { + this.parseHttpHeaders = parse; + } + + public String getFileExtension() { + return ARC_FILE_EXTENSION; + } + + public String getDotFileExtension() { + return DOT_ARC_FILE_EXTENSION; + } + + protected boolean output(final String format) + throws IOException, java.text.ParseException { + boolean result = super.output(format); + if(!result && (format.equals(NOHEAD) || format.equals(HEADER))) { + throw new IOException(format + + " format only supported for single Records"); + } + return result; + } + + public boolean outputRecord(final String format) throws IOException { + boolean result = super.outputRecord(format); + if (result) { + return result; + } + if (format.equals(NOHEAD)) { + // No point digesting if dumping content. + setDigest(false); + ARCRecord r = (ARCRecord) get(); + r.skipHttpHeader(); + r.dump(); + result = true; + } else if (format.equals(HEADER)) { + // No point digesting if dumping content. + setDigest(false); + ARCRecord r = (ARCRecord) get(); + r.dumpHttpHeader(); + result = true; + } + + return result; + } + + public void dump(final boolean compress) + throws IOException, java.text.ParseException { + // No point digesting if we're doing a dump. + setDigest(false); + boolean firstRecord = true; + ARCWriter writer = null; + for (Iterator ii = iterator(); ii.hasNext();) { + ARCRecord r = (ARCRecord)ii.next(); + // We're to dump the arc on stdout. + // Get the first record's data if any. + ARCRecordMetaData meta = r.getMetaData(); + if (firstRecord) { + firstRecord = false; + // Get an ARCWriter. + ByteArrayOutputStream baos = + new ByteArrayOutputStream(r.available()); + // This is slow but done only once at top of ARC. + while (r.available() > 0) { + baos.write(r.read()); + } + List listOfMetadata = new ArrayList(); + listOfMetadata.add(baos.toString(WriterPoolMember.UTF8)); + // Assume getArc returns full path to file. ARCWriter + // or new File will complain if it is otherwise. + writer = new ARCWriter(new AtomicInteger(), System.out, + new File(meta.getArc()), + compress, meta.getDate(), listOfMetadata); + continue; + } + + writer.write(meta.getUrl(), meta.getMimetype(), meta.getIp(), + ArchiveUtils.parse14DigitDate(meta.getDate()).getTime(), + (int)meta.getLength(), r); + } + // System.out.println(System.currentTimeMillis() - start); + } + + /** + * @return an ArchiveReader that will delete a local file on close. Used + * when we bring Archive files local and need to clean up afterward. + */ + public ARCReader getDeleteFileOnCloseReader(final File f) { + final ARCReader d = this; + return new ARCReader() { + private final ARCReader delegate = d; + private File archiveFile = f; + + public void close() throws IOException { + this.delegate.close(); + if (this.archiveFile != null) { + if (archiveFile.exists()) { + archiveFile.delete(); + } + this.archiveFile = null; + } + } + + public ArchiveRecord get(long o) throws IOException { + return this.delegate.get(o); + } + + public boolean isDigest() { + return this.delegate.isDigest(); + } + + public boolean isStrict() { + return this.delegate.isStrict(); + } + + public Iterator iterator() { + return this.delegate.iterator(); + } + + public void setDigest(boolean d) { + this.delegate.setDigest(d); + } + + public void setStrict(boolean s) { + this.delegate.setStrict(s); + } + + public List validate() throws IOException { + return this.delegate.validate(); + } + + @Override + public ArchiveRecord get() throws IOException { + return this.delegate.get(); + } + + @Override + public String getVersion() { + return this.delegate.getVersion(); + } + + @Override + public List validate(int noRecords) throws IOException { + return this.delegate.validate(noRecords); + } + + @Override + protected ARCRecord createArchiveRecord(InputStream is, + long offset) + throws IOException { + return this.delegate.createArchiveRecord(is, offset); + } + + @Override + protected void gotoEOR(ArchiveRecord record) throws IOException { + this.delegate.gotoEOR(record); + } + + @Override + public void dump(boolean compress) + throws IOException, java.text.ParseException { + this.delegate.dump(compress); + } + + @Override + public String getDotFileExtension() { + return this.delegate.getDotFileExtension(); + } + + @Override + public String getFileExtension() { + return this.delegate.getFileExtension(); + } + }; + } + + // Static methods follow. + + /** + * + * @param formatter Help formatter instance. + * @param options Usage options. + * @param exitCode Exit code. + */ + private static void usage(HelpFormatter formatter, Options options, + int exitCode) { + formatter.printHelp("java org.archive.io.arc.ARCReader" + + " [--digest=true|false] \\\n" + + " [--format=cdx|cdxfile|dump|gzipdump|header|nohead]" + + " [--offset=#] \\\n[--strict] [--parse] ARC_FILE|ARC_URL", + options); + System.exit(exitCode); + } + + /** + * Write out the arcfile. + * + * @param reader + * @param format Format to use outputting. + * @throws IOException + * @throws java.text.ParseException + */ + protected static void output(ARCReader reader, String format) + throws IOException, java.text.ParseException { + if (!reader.output(format)) { + throw new IOException("Unsupported format: " + format); + } + } + + /** + * Generate a CDX index file for an ARC file. + * + * @param urlOrPath The ARC file to generate a CDX index for + * @throws IOException + * @throws java.text.ParseException + */ + public static void createCDXIndexFile(String urlOrPath) + throws IOException, java.text.ParseException { + ARCReader r = ARCReaderFactory.get(urlOrPath); + r.setStrict(false); + r.setParseHttpHeaders(true); + r.setDigest(true); + output(r, CDX_FILE); + } + + /** + * Command-line interface to ARCReader. + * + * Here is the command-line interface: + * 

+     * usage: java org.archive.io.arc.ARCReader [--offset=#] ARCFILE
+     *  -h,--help      Prints this message and exits.
+     *  -o,--offset    Outputs record at this offset into arc file.
+ * + *

See in $HERITRIX_HOME/bin/arcreader for a script that'll + * take care of classpaths and the calling of ARCReader. + * + *

Outputs using a pseudo-CDX format as described here: + * CDX + * Legent and here + * Example. + * Legend used in below is: 'CDX b e a m s c V (or v if uncompressed) n g'. + * Hash is hard-coded straight SHA-1 hash of content. + * + * @param args Command-line arguments. + * @throws ParseException Failed parse of the command line. + * @throws IOException + * @throws java.text.ParseException + */ + @SuppressWarnings("unchecked") + public static void main(String [] args) + throws ParseException, IOException, java.text.ParseException { + Options options = getOptions(); + options.addOption(new Option("p","parse", false, "Parse headers.")); + PosixParser parser = new PosixParser(); + CommandLine cmdline = parser.parse(options, args, false); + List cmdlineArgs = cmdline.getArgList(); + Option [] cmdlineOptions = cmdline.getOptions(); + HelpFormatter formatter = new HelpFormatter(); + + // If no args, print help. + if (cmdlineArgs.size() <= 0) { + usage(formatter, options, 0); + } + + // Now look at options passed. + long offset = -1; + boolean digest = false; + boolean strict = false; + boolean parse = false; + String format = CDX; + for (int i = 0; i < cmdlineOptions.length; i++) { + switch(cmdlineOptions[i].getId()) { + case 'h': + usage(formatter, options, 0); + break; + + case 'o': + offset = + Long.parseLong(cmdlineOptions[i].getValue()); + break; + + case 's': + strict = true; + break; + + case 'p': + parse = true; + break; + + case 'd': + digest = getTrueOrFalse(cmdlineOptions[i].getValue()); + break; + + case 'f': + format = cmdlineOptions[i].getValue().toLowerCase(); + boolean match = false; + // List of supported formats. + final String [] supportedFormats = + {CDX, DUMP, GZIP_DUMP, HEADER, NOHEAD, CDX_FILE}; + for (int ii = 0; ii < supportedFormats.length; ii++) { + if (supportedFormats[ii].equals(format)) { + match = true; + break; + } + } + if (!match) { + usage(formatter, options, 1); + } + break; + + default: + throw new RuntimeException("Unexpected option: " + + + cmdlineOptions[i].getId()); + } + } + + if (offset >= 0) { + if (cmdlineArgs.size() != 1) { + System.out.println("Error: Pass one arcfile only."); + usage(formatter, options, 1); + } + ARCReader arc = ARCReaderFactory.get((String)cmdlineArgs.get(0), + offset); + arc.setStrict(strict); + // We must parse headers if we need to skip them. + if (format.equals(NOHEAD) || format.equals(HEADER)) { + parse = true; + } + arc.setParseHttpHeaders(parse); + outputRecord(arc, format); + } else { + for (Iterator i = cmdlineArgs.iterator(); i.hasNext();) { + String urlOrPath = (String)i.next(); + try { + ARCReader r = ARCReaderFactory.get(urlOrPath); + r.setStrict(strict); + r.setParseHttpHeaders(parse); + r.setDigest(digest); + output(r, format); + } catch (RuntimeException e) { + // Write out name of file we failed on to help with + // debugging. Then print stack trace and try to keep + // going. We do this for case where we're being fed + // a bunch of ARCs; just note the bad one and move + // on to the next. + System.err.println("Exception processing " + urlOrPath + + ": " + e.getMessage()); + e.printStackTrace(System.err); + System.exit(1); + } + } + } + } +} diff --git a/commons/src/main/java/org/archive/io/arc/ARCReaderFactory.java b/commons/src/main/java/org/archive/io/arc/ARCReaderFactory.java new file mode 100644 index 00000000..c05fb6b5 --- /dev/null +++ b/commons/src/main/java/org/archive/io/arc/ARCReaderFactory.java @@ -0,0 +1,427 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io.arc; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Iterator; +import java.util.logging.Level; + +import org.archive.io.ArchiveReader; +import org.archive.io.ArchiveReaderFactory; +import org.archive.io.ArchiveRecord; +import org.archive.io.ArchiveRecordHeader; +import org.archive.io.GzipHeader; +import org.archive.io.GzippedInputStream; +import org.archive.io.NoGzipMagicException; +import org.archive.util.FileUtils; + + +/** + * Factory that returns an ARCReader. + * + * Can handle compressed and uncompressed ARCs. + * + * @author stack + */ +public class ARCReaderFactory extends ArchiveReaderFactory +implements ARCConstants { + /** + * This factory instance. + */ + private static final ARCReaderFactory factory = new ARCReaderFactory(); + + /** + * Shutdown any access to default constructor. + */ + protected ARCReaderFactory() { + super(); + } + + public static ARCReader get(String arcFileOrUrl) + throws MalformedURLException, IOException { + return (ARCReader)ARCReaderFactory.factory. + getArchiveReader(arcFileOrUrl); + } + + public static ARCReader get(String arcFileOrUrl, final long offset) + throws MalformedURLException, IOException { + return (ARCReader)ARCReaderFactory.factory. + getArchiveReader(arcFileOrUrl, offset); + } + + public static ARCReader get(final File f) throws IOException { + return (ARCReader)ARCReaderFactory.factory.getArchiveReader(f); + } + + public static ARCReader get(final File f, final long offset) + throws IOException { + return (ARCReader)ARCReaderFactory.factory.getArchiveReader(f, offset); + } + + protected ArchiveReader getArchiveReader(final File f, final long offset) + throws IOException { + return getArchiveReader(f, true, offset); + } + + /** + * @param f An arcfile to read. + * @param skipSuffixTest Set to true if want to test that ARC has proper + * suffix. Use this method and pass false to open ARCs + * with the .open or otherwise suffix. + * @param offset Have returned ARCReader set to start reading at passed + * offset. + * @return An ARCReader. + * @throws IOException + */ + public static ARCReader get(final File f, + final boolean skipSuffixTest, final long offset) + throws IOException { + return (ARCReader)ARCReaderFactory.factory.getArchiveReader(f, + skipSuffixTest, offset); + } + + protected ArchiveReader getArchiveReader(final File arcFile, + final boolean skipSuffixTest, final long offset) + throws IOException { + boolean compressed = testCompressedARCFile(arcFile, skipSuffixTest); + if (!compressed) { + if (!FileUtils.isReadableWithExtensionAndMagic(arcFile, + ARC_FILE_EXTENSION, ARC_MAGIC_NUMBER)) { + throw new IOException(arcFile.getAbsolutePath() + + " is not an Internet Archive ARC file."); + } + } + return compressed? + (ARCReader)ARCReaderFactory.factory. + new CompressedARCReader(arcFile, offset): + (ARCReader)ARCReaderFactory.factory. + new UncompressedARCReader(arcFile, offset); + } + + public static ArchiveReader get(final String s, final InputStream is, + final boolean atFirstRecord) + throws IOException { + return ARCReaderFactory.factory.getArchiveReader(s, is, + atFirstRecord); + } + + protected ArchiveReader getArchiveReader(final String arc, + final InputStream is, final boolean atFirstRecord) + throws IOException { + // For now, assume stream is compressed. Later add test of input + // stream or handle exception thrown when figure not compressed stream. + return new CompressedARCReader(arc, asRepositionable(is), + atFirstRecord); + } + + /** + * Get an ARCReader aligned at offset. This version of get + * will not bring the ARC local but will try to stream across the net making + * an HTTP 1.1 Range request on remote http server (RFC1435 Section 14.35). + * + * @param arcUrl HTTP URL for an ARC (All ARCs considered remote). + * @param offset Offset into ARC at which to start fetching. + * @return An ARCReader aligned at offset. + * @throws IOException + */ + public static ARCReader get(final URL arcUrl, final long offset) + throws IOException { + return (ARCReader)ARCReaderFactory.factory.getArchiveReader(arcUrl, + offset); + } + + /** + * Get an ARCReader. + * Pulls the ARC local into whereever the System Property + * java.io.tmpdir points. It then hands back an ARCReader that + * points at this local copy. A close on this ARCReader instance will + * remove the local copy. + * @param arcUrl An URL that points at an ARC. + * @return An ARCReader. + * @throws IOException + */ + public static ARCReader get(final URL arcUrl) + throws IOException { + return (ARCReader)ARCReaderFactory.factory.getArchiveReader(arcUrl); + } + + /** + * @param arcFile File to test. + * @return True if arcFile is compressed ARC. + * @throws IOException + */ + public boolean isCompressed(File arcFile) throws IOException { + return testCompressedARCFile(arcFile); + } + + /** + * Check file is compressed and in ARC GZIP format. + * + * @param arcFile File to test if its Internet Archive ARC file + * GZIP compressed. + * + * @return True if this is an Internet Archive GZIP'd ARC file (It begins + * w/ the Internet Archive GZIP header and has the + * COMPRESSED_ARC_FILE_EXTENSION suffix). + * + * @exception IOException If file does not exist or is not unreadable. + */ + public static boolean testCompressedARCFile(File arcFile) + throws IOException { + return testCompressedARCFile(arcFile, false); + } + + /** + * Check file is compressed and in ARC GZIP format. + * + * @param arcFile File to test if its Internet Archive ARC file + * GZIP compressed. + * @param skipSuffixCheck Set to true if we're not to test on the + * '.arc.gz' suffix. + * + * @return True if this is an Internet Archive GZIP'd ARC file (It begins + * w/ the Internet Archive GZIP header). + * + * @exception IOException If file does not exist or is not unreadable. + */ + public static boolean testCompressedARCFile(File arcFile, + boolean skipSuffixCheck) + throws IOException { + boolean compressedARCFile = false; + FileUtils.assertReadable(arcFile); + if(!skipSuffixCheck && !arcFile.getName().toLowerCase() + .endsWith(COMPRESSED_ARC_FILE_EXTENSION)) { + return compressedARCFile; + } + + final InputStream is = new FileInputStream(arcFile); + try { + compressedARCFile = testCompressedARCStream(is); + } finally { + is.close(); + } + return compressedARCFile; + } + + public static boolean isARCSuffix(final String arcName) { + return (arcName == null)? + false: + (arcName.toLowerCase().endsWith(DOT_COMPRESSED_ARC_FILE_EXTENSION))? + true: + (arcName.toLowerCase().endsWith(DOT_ARC_FILE_EXTENSION))? + true: false; + } + + /** + * Tests passed stream is gzip stream by reading in the HEAD. + * Does not reposition the stream. That is left up to the caller. + * @param is An InputStream. + * @return True if compressed stream. + * @throws IOException + */ + public static boolean testCompressedARCStream(final InputStream is) + throws IOException { + boolean compressedARCFile = false; + GzipHeader gh = null; + try { + gh = new GzipHeader(is); + } catch (NoGzipMagicException e ) { + return compressedARCFile; + } + + byte[] fextra = gh.getFextra(); + // Now make sure following bytes are IA GZIP comment. + // First check length. ARC_GZIP_EXTRA_FIELD includes length + // so subtract two and start compare to ARC_GZIP_EXTRA_FIELD + // at +2. + if (fextra != null && + ARC_GZIP_EXTRA_FIELD.length - 2 == fextra.length) { + compressedARCFile = true; + for (int i = 0; i < fextra.length; i++) { + if (fextra[i] != ARC_GZIP_EXTRA_FIELD[i + 2]) { + compressedARCFile = false; + break; + } + } + } + return compressedARCFile; + } + + /** + * Uncompressed arc file reader. + * @author stack + */ + private class UncompressedARCReader extends ARCReader { + /** + * Constructor. + * @param f Uncompressed arcfile to read. + * @throws IOException + */ + public UncompressedARCReader(final File f) + throws IOException { + this(f, 0); + } + + /** + * Constructor. + * + * @param f Uncompressed arcfile to read. + * @param offset Offset at which to position ARCReader. + * @throws IOException + */ + public UncompressedARCReader(final File f, final long offset) + throws IOException { + // Arc file has been tested for existence by time it has come + // to here. + setIn(getInputStream(f, offset)); + initialize(f.getAbsolutePath()); + } + + /** + * Constructor. + * + * @param f Uncompressed arc to read. + * @param is InputStream. + */ + public UncompressedARCReader(final String f, final InputStream is) { + // Arc file has been tested for existence by time it has come + // to here. + setIn(is); + initialize(f); + } + } + + /** + * Compressed arc file reader. + * + * @author stack + */ + private class CompressedARCReader extends ARCReader { + + /** + * Constructor. + * + * @param f + * Compressed arcfile to read. + * @throws IOException + */ + public CompressedARCReader(final File f) throws IOException { + this(f, 0); + } + + /** + * Constructor. + * + * @param f Compressed arcfile to read. + * @param offset Position at where to start reading file. + * @throws IOException + */ + public CompressedARCReader(final File f, final long offset) + throws IOException { + // Arc file has been tested for existence by time it has come + // to here. + setIn(new GzippedInputStream(getInputStream(f, offset))); + setCompressed((offset == 0)); + initialize(f.getAbsolutePath()); + } + + /** + * Constructor. + * + * @param f Compressed arcfile. + * @param is InputStream to use. + * @throws IOException + */ + public CompressedARCReader(final String f, final InputStream is, + final boolean atFirstRecord) + throws IOException { + // Arc file has been tested for existence by time it has come + // to here. + setIn(new GzippedInputStream(is)); + setCompressed(true); + setAlignedOnFirstRecord(atFirstRecord); + initialize(f); + } + + /** + * Get record at passed offset. + * + * @param offset + * Byte index into arcfile at which a record starts. + * @return An ARCRecord reference. + * @throws IOException + */ + public ARCRecord get(long offset) throws IOException { + cleanupCurrentRecord(); + ((GzippedInputStream)getIn()).gzipMemberSeek(offset); + return createArchiveRecord(getIn(), offset); + } + + public Iterator iterator() { + /** + * Override ARCRecordIterator so can base returned iterator on + * GzippedInputStream iterator. + */ + return new ArchiveRecordIterator() { + private GzippedInputStream gis = + (GzippedInputStream)getInputStream(); + + private Iterator gzipIterator = this.gis.iterator(); + + protected boolean innerHasNext() { + return this.gzipIterator.hasNext(); + } + + protected ArchiveRecord innerNext() throws IOException { + // Get the position before gzipIterator.next moves + // it on past the gzip header. + long p = this.gis.position(); + InputStream is = (InputStream) this.gzipIterator.next(); + return createArchiveRecord(is, p); + } + }; + } + + protected void gotoEOR(ArchiveRecord rec) throws IOException { + long skipped = ((GzippedInputStream)getIn()). + gotoEOR(LINE_SEPARATOR); + if (skipped <= 0) { + return; + } + // Report on system error the number of unexpected characters + // at the end of this record. + ArchiveRecordHeader meta = (getCurrentRecord() != null)? + rec.getHeader(): null; + String message = "Record ENDING at " + + ((GzippedInputStream)getIn()).position() + + " has " + skipped + " trailing byte(s): " + + ((meta != null)? meta.toString(): ""); + if (isStrict()) { + throw new IOException(message); + } + logStdErr(Level.WARNING, message); + } + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/io/arc/ARCRecord.java b/commons/src/main/java/org/archive/io/arc/ARCRecord.java new file mode 100644 index 00000000..3e5b73ea --- /dev/null +++ b/commons/src/main/java/org/archive/io/arc/ARCRecord.java @@ -0,0 +1,718 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io.arc; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.regex.Matcher; + +import org.apache.commons.httpclient.Header; +import org.apache.commons.httpclient.HttpParser; +import org.apache.commons.httpclient.StatusLine; +import org.apache.commons.httpclient.util.EncodingUtil; +import org.archive.io.ArchiveRecord; +import org.archive.io.ArchiveRecordHeader; +import org.archive.io.RecoverableIOException; +import org.archive.util.InetAddressUtil; +import org.archive.util.TextUtils; + +/** + * An ARC file record. + * Does not compass the ARCRecord metadata line, just the record content. + * @author stack + */ +public class ARCRecord extends ArchiveRecord implements ARCConstants { + /** + * Http status line object. + * + * May be null if record is not http. + */ + private StatusLine httpStatus = null; + + /** + * Http header bytes. + * + * If non-null and bytes available, give out its contents before we + * go back to the underlying stream. + */ + private InputStream httpHeaderStream = null; + + /** + * Http headers. + * + * Only populated after reading of headers. + */ + private Header [] httpHeaders = null; + + /** + * Array of field names. + * + * Used to initialize headerFieldNameKeys. + */ + private final String [] headerFieldNameKeysArray = { + URL_FIELD_KEY, + IP_HEADER_FIELD_KEY, + DATE_FIELD_KEY, + MIMETYPE_FIELD_KEY, + LENGTH_FIELD_KEY + }; + + /** + * An array of the header field names found in the ARC file header on + * the 3rd line. + * + * We used to read these in from the arc file first record 3rd line but + * now we hardcode them for sake of improved performance. + */ + private final List headerFieldNameKeys = + Arrays.asList(this.headerFieldNameKeysArray); + + /** + * Constructor. + * + * @param in Stream cue'd up to be at the start of the record this instance + * is to represent. + * @param metaData Meta data. + * @throws IOException + */ + public ARCRecord(InputStream in, ArchiveRecordHeader metaData) + throws IOException { + this(in, metaData, 0, true, false, true); + } + + /** + * Constructor. + * + * @param in Stream cue'd up to be at the start of the record this instance + * is to represent. + * @param metaData Meta data. + * @param bodyOffset Offset into the body. Usually 0. + * @param digest True if we're to calculate digest for this record. Not + * digesting saves about ~15% of cpu during an ARC parse. + * @param strict Be strict parsing (Parsing stops if ARC inproperly + * formatted). + * @param parseHttpHeaders True if we are to parse HTTP headers. Costs + * about ~20% of CPU during an ARC parse. + * @throws IOException + */ + public ARCRecord(InputStream in, ArchiveRecordHeader metaData, + int bodyOffset, boolean digest, boolean strict, + final boolean parseHttpHeaders) + throws IOException { + super(in, metaData, bodyOffset, digest, strict); + if (parseHttpHeaders) { + this.httpHeaderStream = readHttpHeader(); + } + } + + /** + * Constructor. + * + * @param in Stream cue'd up to be at the start of the records metadata + * this instance is to represent. + * @param identifier Identifier for this the hosting Reader. + * @param offset Current offset into in (Used to keep + * position properly aligned). Usually 0. + * @param digest True if we're to calculate digest for this record. Not + * digesting saves about ~15% of cpu during an ARC parse. + * @param strict Be strict parsing (Parsing stops if ARC inproperly + * formatted). + * @param parseHttpHeaders True if we are to parse HTTP headers. Costs + * about ~20% of CPU during an ARC parse. + * @param isAllignedOnFirstRecord True if this is the first record to be + * read from an archive + * @param String version Version information to be returned to the + * ARCReader constructing this record + * + * @throws IOException + */ + public ARCRecord(InputStream in, final String identifier, + final long offset, boolean digest, boolean strict, + final boolean parseHttpHeaders, + final boolean isAlignedOnFirstRecord, String version) + throws IOException { + super(in, null, 0, digest, strict); + setHeader(parseHeaders(in, identifier, offset, strict, isAlignedOnFirstRecord, version)); + if (parseHttpHeaders) { + this.httpHeaderStream = readHttpHeader(); + } + } + + /** + * Constructor. + * + * @param in Stream cue'd up to be at the start of the records metadata + * this instance is to represent. + * @param identifier Identifier for this the hosting Reader. + * @param offset Current offset into in (Used to keep + * position properly aligned). Usually 0. + * @param digest True if we're to calculate digest for this record. Not + * digesting saves about ~15% of cpu during an ARC parse. + * @param strict Be strict parsing (Parsing stops if ARC inproperly + * formatted). + * @param parseHttpHeaders True if we are to parse HTTP headers. Costs + * about ~20% of CPU during an ARC parse. + * + * @throws IOException + */ + public ARCRecord(InputStream in, final String identifier, + final long offset, boolean digest, boolean strict, + final boolean parseHttpHeaders) + throws IOException { + this(in, identifier, offset, digest, strict, parseHttpHeaders, + false, null); + } + + private ArchiveRecordHeader parseHeaders(final InputStream in, + final String identifier, final long offset, final boolean strict, + final boolean isAlignedOnFirstRecord, String version) + throws IOException { + + ArrayList firstLineValues = new ArrayList(20); + getTokenizedHeaderLine(in, firstLineValues); + + int bodyOffset = 0; + if (offset == 0 && isAlignedOnFirstRecord) { + // If offset is zero and we were aligned at first record on + // creation (See #alignedOnFirstRecord for more on this), then no + // records have been read yet and we're reading our first one, the + // record of ARC file meta info. Its special. In ARC versions + // 1.x, first record has three lines of meta info. We've just read + // the first line. There are two more. The second line has misc. + // info. We're only interested in the first field, the version + // number. The third line is the list of field names. Here's what + // ARC file version 1.x meta content looks like: + // + // filedesc://testIsBoundary-JunitIAH200401070157520.arc 0.0.0.0 \\ + // 20040107015752 text/plain 77 + // 1 0 InternetArchive + // URL IP-address Archive-date Content-type Archive-length + // + ArrayList secondLineValues = new ArrayList(20); + bodyOffset += getTokenizedHeaderLine(in, secondLineValues); + version = ((String)secondLineValues.get(0) + + "." + (String)secondLineValues.get(1)); + // Just read over the 3rd line. We used to parse it and use + // values found here but now we just hardcode them to avoid + // having to read this 3rd line even for random arc file accesses. + bodyOffset += getTokenizedHeaderLine(in, null); + // this.position = bodyOffset; + } + setBodyOffset(bodyOffset); + + return computeMetaData(this.headerFieldNameKeys, firstLineValues, version, offset, identifier); + } + + /** + * Get a record header line as list of tokens. + * + * We keep reading till we find a LINE_SEPARATOR or we reach the end + * of file w/o finding a LINE_SEPARATOR or the line length is crazy. + * + * @param stream InputStream to read from. + * @param list Empty list that gets filled w/ string tokens. + * @return Count of characters read. + * @exception IOException If problem reading stream or no line separator + * found or EOF before EOL or we didn't get minimum header fields. + */ + private int getTokenizedHeaderLine(final InputStream stream, + List list) throws IOException { + // Preallocate usual line size. + StringBuilder buffer = new StringBuilder(2048 + 20); + int read = 0; + int previous = -1; + for (int c = -1; true;) { + previous = c; + c = stream.read(); + if (c == -1) { + throw new RecoverableIOException("Hit EOF before header EOL."); + } + c &= 0xff; + read++; + if (read > MAX_HEADER_LINE_LENGTH) { + throw new IOException("Header line longer than max allowed " + + " -- " + String.valueOf(MAX_HEADER_LINE_LENGTH) + + " -- or passed buffer doesn't contain a line (Read: " + + buffer.length() + "). Here's" + + " some of what was read: " + + buffer.substring(0, Math.min(buffer.length(), 256))); + } + + if (c == LINE_SEPARATOR) { + if (buffer.length() == 0) { + // Empty line at start of buffer. Skip it and try again. + continue; + } + + if (list != null) { + list.add(buffer.toString()); + } + // LOOP TERMINATION. + break; + } else if (c == HEADER_FIELD_SEPARATOR) { + if (!isStrict() && previous == HEADER_FIELD_SEPARATOR) { + // Early ARCs sometimes had multiple spaces between fields. + continue; + } + if (list != null) { + list.add(buffer.toString()); + } + // reset to empty + buffer.setLength(0); + } else { + buffer.append((char)c); + } + } + + // List must have at least 3 elements in it and no more than 10. If + // it has other than this, then bogus parse. + if (list != null && (list.size() < 3 || list.size() > 100)) { + throw new IOException("Unparseable header line: " + list); + } + + return read; + } + + /** + * Compute metadata fields. + * + * Here we check the meta field has right number of items in it. + * + * @param keys Keys to use composing headerFields map. + * @param values Values to set into the headerFields map. + * @param v The version of this ARC file. + * @param offset Offset into arc file. + * + * @return Metadata structure for this record. + * + * @exception IOException If no. of keys doesn't match no. of values. + */ + private ARCRecordMetaData computeMetaData(List keys, + List values, String v, long offset, final String identifier) + throws IOException { + if (keys.size() != values.size()) { + List originalValues = values; + if (!isStrict()) { + values = fixSpaceInURL(values, keys.size()); + // If values still doesn't match key size, try and do + // further repair. + if (keys.size() != values.size()) { + // Early ARCs had a space in mimetype. + if (values.size() == (keys.size() + 1) && + values.get(4).toLowerCase().startsWith("charset=")) { + List nuvalues = + new ArrayList(keys.size()); + nuvalues.add(0, values.get(0)); + nuvalues.add(1, values.get(1)); + nuvalues.add(2, values.get(2)); + nuvalues.add(3, values.get(3) + values.get(4)); + nuvalues.add(4, values.get(5)); + values = nuvalues; + } else if((values.size() + 1) == keys.size() && + isLegitimateIPValue(values.get(1)) && + isDate(values.get(2)) && isNumber(values.get(3))) { + // Mimetype is empty. + List nuvalues = + new ArrayList(keys.size()); + nuvalues.add(0, values.get(0)); + nuvalues.add(1, values.get(1)); + nuvalues.add(2, values.get(2)); + nuvalues.add(3, "-"); + nuvalues.add(4, values.get(3)); + values = nuvalues; + } + } + } + if (keys.size() != values.size()) { + throw new IOException("Size of field name keys does" + + " not match count of field values: " + values); + } + // Note that field was fixed on stderr. + System.err.println(Level.WARNING.toString() + "Fixed spaces in metadata line at " + + "offset " + offset + + " Original: " + originalValues + ", New: " + values); + } + + Map headerFields = + new HashMap(keys.size() + 2); + for (int i = 0; i < keys.size(); i++) { + headerFields.put(keys.get(i), values.get(i)); + } + + // Add a check for tabs in URLs. If any, replace with '%09'. + // See https://sourceforge.net/tracker/?group_id=73833&atid=539099&func=detail&aid=1010966, + // [ 1010966 ] crawl.log has URIs with spaces in them. + String url = (String)headerFields.get(URL_FIELD_KEY); + if (url != null && url.indexOf('\t') >= 0) { + headerFields.put(URL_FIELD_KEY, + TextUtils.replaceAll("\t", url, "%09")); + } + + headerFields.put(VERSION_FIELD_KEY, v); + headerFields.put(ABSOLUTE_OFFSET_KEY, new Long(offset)); + + return new ARCRecordMetaData(identifier, headerFields); + } + + /** + * Fix space in URLs. + * The ARCWriter used to write into the ARC URLs with spaces in them. + * See [ 1010966 ] + * crawl.log has URIs with spaces in them. + * This method does fix up on such headers converting all spaces found + * to '%20'. + * @param values List of metadata values. + * @param requiredSize Expected size of resultant values list. + * @return New list if we successfully fixed up values or original if + * fixup failed. + */ + private List fixSpaceInURL(List values, int requiredSize) { + // Do validity check. 3rd from last is a date of 14 numeric + // characters. The 4th from last is IP, all before the IP + // should be concatenated together with a '%20' joiner. + // In the below, '4' is 4th field from end which has the IP. + if (!(values.size() > requiredSize) || values.size() < 4) { + return values; + } + // Test 3rd field is valid date. + if (!isDate((String) values.get(values.size() - 3))) { + return values; + } + + // Test 4th field is valid IP. + if (!isLegitimateIPValue((String) values.get(values.size() - 4))) { + return values; + } + + List newValues = new ArrayList(requiredSize); + StringBuffer url = new StringBuffer(); + for (int i = 0; i < (values.size() - 4); i++) { + if (i > 0) { + url.append("%20"); + } + url.append(values.get(i)); + } + newValues.add(url.toString()); + for (int i = values.size() - 4; i < values.size(); i++) { + newValues.add(values.get(i)); + } + return newValues; + } + + private boolean isDate(final String date) { + if (date.length() != 14) { + return false; + } + return isNumber(date); + } + + private boolean isNumber(final String n) { + for (int i = 0; i < n.length(); i++) { + if (!Character.isDigit(n.charAt(i))) { + return false; + } + } + return true; + } + + private boolean isLegitimateIPValue(final String ip) { + if ("-".equals(ip)) { + return true; + } + Matcher m = InetAddressUtil.IPV4_QUADS.matcher(ip); + return m != null && m.matches(); + } + + /** + * Skip over the the http header if one present. + * + * Subsequent reads will get the body. + * + *

Calling this method in the midst of reading the header + * will make for strange results. Otherwise, safe to call + * at any time though before reading any of the arc record + * content is only time that it makes sense. + * + *

After calling this method, you can call + * {@link #getHttpHeaders()} to get the read http header. + * + * @throws IOException + */ + public void skipHttpHeader() throws IOException { + if (this.httpHeaderStream != null) { + // Empty the httpHeaderStream + for (int available = this.httpHeaderStream.available(); + this.httpHeaderStream != null && + (available = this.httpHeaderStream.available()) > 0;) { + // We should be in this loop once only we should only do this + // buffer allocation once. + byte [] buffer = new byte[available]; + // The read nulls out httpHeaderStream when done with it so + // need check for null in the loop control line. + read(buffer, 0, available); + } + } + } + + public void dumpHttpHeader() throws IOException { + if (this.httpHeaderStream == null) { + return; + } + // Dump the httpHeaderStream to STDOUT + for (int available = this.httpHeaderStream.available(); + this.httpHeaderStream != null + && (available = this.httpHeaderStream.available()) > 0;) { + // We should be in this loop only once and should do this + // buffer allocation once. + byte[] buffer = new byte[available]; + // The read nulls out httpHeaderStream when done with it so + // need check for null in the loop control line. + int read = read(buffer, 0, available); + System.out.write(buffer, 0, read); + } + } + + /** + * Read http header if present. Technique borrowed from HttpClient HttpParse + * class. + * + * @return ByteArrayInputStream with the http header in it or null if no + * http header. + * @throws IOException + */ + private InputStream readHttpHeader() throws IOException { + // If judged a record that doesn't have an http header, return + // immediately. + String url = getHeader().getUrl(); + if(!url.startsWith("http") || + getHeader().getLength() <= MIN_HTTP_HEADER_LENGTH) { + return null; + } + byte [] statusBytes = HttpParser.readRawLine(getIn()); + int eolCharCount = getEolCharsCount(statusBytes); + if (eolCharCount <= 0) { + throw new RecoverableIOException("Failed to read http status where one " + + " was expected: " + new String(statusBytes)); + } + String statusLine = EncodingUtil.getString(statusBytes, 0, + statusBytes.length - eolCharCount, ARCConstants.DEFAULT_ENCODING); + if ((statusLine == null) || + !StatusLine.startsWithHTTP(statusLine)) { + if (statusLine.startsWith("DELETED")) { + // Some old ARCs have deleted records like following: + // http://vireo.gatech.edu:80/ebt-bin/nph-dweb/dynaweb/SGI_Developer/SGITCL_PG/@Generic__BookTocView/11108%3Btd%3D2 130.207.168.42 19991010131803 text/html 29202 + // DELETED_TIME=20000425001133_DELETER=Kurt_REASON=alexalist + // (follows ~29K spaces) + // For now, throw a RecoverableIOException so if iterating over + // records, we keep going. TODO: Later make a legitimate + // ARCRecord from the deleted record rather than throw + // exception. + throw new DeletedARCRecordIOException(statusLine); + } else { + throw new RecoverableIOException("Failed parse of http status line."); + } + } + this.httpStatus = new StatusLine(statusLine); + + // Save off all bytes read. Keep them as bytes rather than + // convert to strings so we don't have to worry about encodings + // though this should never be a problem doing http headers since + // its all supposed to be ascii. + ByteArrayOutputStream baos = + new ByteArrayOutputStream(statusBytes.length + 4 * 1024); + baos.write(statusBytes); + + // Now read rest of the header lines looking for the separation + // between header and body. + for (byte [] lineBytes = null; true;) { + lineBytes = HttpParser.readRawLine(getIn()); + eolCharCount = getEolCharsCount(lineBytes); + if (eolCharCount <= 0) { + throw new IOException("Failed reading http headers: " + + ((lineBytes != null)? new String(lineBytes): null)); + } + // Save the bytes read. + baos.write(lineBytes); + if ((lineBytes.length - eolCharCount) <= 0) { + // We've finished reading the http header. + break; + } + } + + byte [] headerBytes = baos.toByteArray(); + // Save off where body starts. + this.getMetaData().setContentBegin(headerBytes.length); + ByteArrayInputStream bais = + new ByteArrayInputStream(headerBytes); + if (!bais.markSupported()) { + throw new IOException("ByteArrayInputStream does not support mark"); + } + bais.mark(headerBytes.length); + // Read the status line. Don't let it into the parseHeaders function. + // It doesn't know what to do with it. + bais.read(statusBytes, 0, statusBytes.length); + this.httpHeaders = HttpParser.parseHeaders(bais, + ARCConstants.DEFAULT_ENCODING); + this.getMetaData().setStatusCode(Integer.toString(getStatusCode())); + bais.reset(); + return bais; + } + + private static class DeletedARCRecordIOException + extends RecoverableIOException { + private static final long serialVersionUID = 1L; + + public DeletedARCRecordIOException(final String reason) { + super(reason); + } + } + + /** + * Return status code for this record. + * + * This method will return -1 until the http header has been read. + * @return Status code. + */ + public int getStatusCode() { + return (this.httpStatus == null)? -1: this.httpStatus.getStatusCode(); + } + + /** + * @param bytes Array of bytes to examine for an EOL. + * @return Count of end-of-line characters or zero if none. + */ + private int getEolCharsCount(byte [] bytes) { + int count = 0; + if (bytes != null && bytes.length >=1 && + bytes[bytes.length - 1] == '\n') { + count++; + if (bytes.length >=2 && bytes[bytes.length -2] == '\r') { + count++; + } + } + return count; + } + + /** + * @return Meta data for this record. + */ + public ARCRecordMetaData getMetaData() { + return (ARCRecordMetaData)getHeader(); + } + + /** + * @return http headers (Only available after header has been read). + */ + public Header [] getHttpHeaders() { + return this.httpHeaders; + } + + /** + * @return Next character in this ARCRecord's content else -1 if at end of + * this record. + * @throws IOException + */ + public int read() throws IOException { + int c = -1; + if (this.httpHeaderStream != null && + (this.httpHeaderStream.available() > 0)) { + // If http header, return bytes from it before we go to underlying + // stream. + c = this.httpHeaderStream.read(); + // If done with the header stream, null it out. + if (this.httpHeaderStream.available() <= 0) { + this.httpHeaderStream = null; + } + incrementPosition(); + } else { + c = super.read(); + } + return c; + } + + public int read(byte [] b, int offset, int length) throws IOException { + int read = -1; + if (this.httpHeaderStream != null && + (this.httpHeaderStream.available() > 0)) { + // If http header, return bytes from it before we go to underlying + // stream. + read = Math.min(length, this.httpHeaderStream.available()); + if (read == 0) { + read = -1; + } else { + read = this.httpHeaderStream.read(b, offset, read); + } + // If done with the header stream, null it out. + if (this.httpHeaderStream.available() <= 0) { + this.httpHeaderStream = null; + } + incrementPosition(read); + } else { + read = super.read(b, offset, length); + } + return read; + } + + /** + * @return Offset at which the body begins (Only known after + * header has been read) or -1 if none or if we haven't read + * headers yet. Usually length of HTTP headers (does not include ARC + * metadata line length). + */ + public int getBodyOffset() { + return this.getMetaData().getContentBegin(); + } + + @Override + protected String getIp4Cdx(ArchiveRecordHeader h) { + String result = null; + if (h instanceof ARCRecordMetaData) { + result = ((ARCRecordMetaData)h).getIp(); + } + return (result != null)? result: super.getIp4Cdx(h); + } + + @Override + protected String getStatusCode4Cdx(ArchiveRecordHeader h) { + String result = null; + if (h instanceof ARCRecordMetaData) { + result = ((ARCRecordMetaData) h).getStatusCode(); + } + return (result != null) ? result: super.getStatusCode4Cdx(h); + } + + @Override + protected String getDigest4Cdx(ArchiveRecordHeader h) { + String result = null; + if (h instanceof ARCRecordMetaData) { + result = ((ARCRecordMetaData) h).getDigest(); + } + return (result != null) ? result: super.getDigest4Cdx(h); + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/io/arc/ARCRecordMetaData.java b/commons/src/main/java/org/archive/io/arc/ARCRecordMetaData.java new file mode 100644 index 00000000..95a3003c --- /dev/null +++ b/commons/src/main/java/org/archive/io/arc/ARCRecordMetaData.java @@ -0,0 +1,259 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io.arc; + +import java.io.File; +import java.io.IOException; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +import org.archive.io.ArchiveRecordHeader; + + +/** + * An immutable class to hold an ARC record meta data. + * + * @author stack + */ +public class ARCRecordMetaData implements ArchiveRecordHeader, ARCConstants { + /** + * Map of record header fields. + * + * We store all in a hashmap. This way we can hold version 1 or + * version 2 record meta data. + * + *

Keys are lowercase. + */ + protected Map headerFields = null; + + /** + * Digest for the record. + * + * Only available after the record has been read in totality. + */ + private String digest = null; + + /** + * Status for this request. + * + * There may be no status. + */ + private String statusCode = null; + + /** + * The arc this metadata came out. + * Descriptive String, either path or URL. + */ + private String arc = null; + + private int contentBegin = 0; + + /** + * Shut down the default constructor. + */ + protected ARCRecordMetaData() { + super(); + } + + /** + * Constructor. + * + * @param arc The arc file this metadata came out of. + * @param headerFields Hash of meta fields. + * + * @throws IOException + */ + public ARCRecordMetaData(final String arc, Map headerFields) + throws IOException { + // Make sure the minimum required fields are present, + for (Iterator i = REQUIRED_VERSION_1_HEADER_FIELDS.iterator(); + i.hasNext(); ) { + testRequiredField(headerFields, (String)i.next()); + } + this.headerFields = headerFields; + this.arc = arc; + } + + /** + * Test required field is present in hash. + * + * @param fields Map of fields. + * @param requiredField Field to test for. + * + * @exception IOException If required field is not present. + */ + protected void testRequiredField(Map fields, String requiredField) + throws IOException { + if (!fields.containsKey(requiredField)) { + throw new IOException("Required field " + requiredField + + " not in meta data."); + } + } + + /** + * Get the time when the record was harvested. + *

+ * Returns the date in Heritrix 14 digit time format (UTC). See the + * {@link org.archive.util.ArchiveUtils} class for converting to Java + * dates. + * + * @return Header date in Heritrix 14 digit format. + * @see org.archive.util.ArchiveUtils#parse14DigitDate(String) + */ + public String getDate() { + return (String) this.headerFields.get(DATE_FIELD_KEY); + } + + /** + * @return Return length of the record. + */ + public long getLength() { + return Long.parseLong((String)this.headerFields. + get(LENGTH_FIELD_KEY)); + } + + /** + * @return Header url. + */ + public String getUrl() { + return (String)this.headerFields.get(URL_FIELD_KEY); + } + + /** + * @return IP. + */ + public String getIp() + { + return (String)this.headerFields.get(IP_HEADER_FIELD_KEY); + } + + /** + * @return mimetype The mimetype that is in the ARC metaline -- NOT the http + * content-type content. + */ + public String getMimetype() { + return (String)this.headerFields.get(MIMETYPE_FIELD_KEY); + } + + /** + * @return Arcfile version. + */ + public String getVersion() { + return (String)this.headerFields.get(VERSION_FIELD_KEY); + } + + /** + * @return Offset into arcfile at which this record begins. + */ + public long getOffset() { + return ((Long)this.headerFields.get(ABSOLUTE_OFFSET_KEY)).longValue(); + } + + /** + * @param key Key to use looking up field value. + * @return value for passed key of null if no such entry. + */ + public Object getHeaderValue(String key) { + return this.headerFields.get(key); + } + + /** + * @return Header field name keys. + */ + public Set getHeaderFieldKeys() + { + return this.headerFields.keySet(); + } + + /** + * @return Map of header fields. + */ + public Map getHeaderFields() { + return this.headerFields; + } + + /** + * @return Returns identifier for ARC. + */ + public String getArc() { + return this.arc; + } + + /** + * @return Convenience method that does a + * return new File(this.arc) (Be aware this.arc is not always + * full path to an ARC file -- may be an URL). Test + * returned file for existence. + */ + public File getArcFile() { + return new File(this.arc); + } + + /** + * @return Returns the digest. + */ + public String getDigest() { + return this.digest; + } + + /** + * @param d The digest to set. + */ + public void setDigest(String d) { + this.digest = d; + } + + /** + * @return Returns the statusCode. May be null. + */ + public String getStatusCode() { + return this.statusCode; + } + + /** + * @param statusCode The statusCode to set. + */ + public void setStatusCode(String statusCode) { + this.statusCode = statusCode; + } + + public String toString() { + return ((this.arc != null)? this.arc: "") + + ": " + + ((this.headerFields != null)? this.headerFields.toString(): ""); + } + + public String getReaderIdentifier() { + return this.getArc(); + } + + public String getRecordIdentifier() { + return getDate() + "/" + getUrl(); + } + + public int getContentBegin() { + return this.contentBegin; + } + + void setContentBegin(final int offset) { + this.contentBegin = offset; + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/io/arc/ARCUtils.java b/commons/src/main/java/org/archive/io/arc/ARCUtils.java new file mode 100644 index 00000000..f39c71c1 --- /dev/null +++ b/commons/src/main/java/org/archive/io/arc/ARCUtils.java @@ -0,0 +1,244 @@ +/* ARCUtils + * + * Created on Aug 10, 2005 + * + * Copyright (C) 2005 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.io.arc; + +import it.unimi.dsi.fastutil.io.RepositionableStream; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; + +import org.archive.io.GzipHeader; +import org.archive.io.NoGzipMagicException; +import org.archive.net.UURI; + +public class ARCUtils implements ARCConstants { + /** + * @param pathOrUri Path or URI to extract arc filename from. + * @return Extracted arc file name. + * @throws URISyntaxException + */ + public static String parseArcFilename(final String pathOrUri) + throws URISyntaxException { + String path = pathOrUri; + if (UURI.hasScheme(pathOrUri)) { + URI url = new URI(pathOrUri); + path = url.getPath(); + } + return (new File(path)).getName(); + } + + /** + * @param arcFile File to test. + * @return True if arcFile is compressed ARC. + * @throws IOException + */ + public static boolean isCompressed(File arcFile) throws IOException { + return testCompressedARCFile(arcFile); + } + + /** + * Check file is compressed and in ARC GZIP format. + * + * @param arcFile File to test if its Internet Archive ARC file + * GZIP compressed. + * + * @return True if this is an Internet Archive GZIP'd ARC file (It begins + * w/ the Internet Archive GZIP header and has the + * COMPRESSED_ARC_FILE_EXTENSION suffix). + * + * @exception IOException If file does not exist or is not unreadable. + */ + public static boolean testCompressedARCFile(File arcFile) + throws IOException { + return testCompressedARCFile(arcFile, false); + } + + /** + * Check file is compressed and in ARC GZIP format. + * + * @param arcFile File to test if its Internet Archive ARC file + * GZIP compressed. + * @param skipSuffixCheck Set to true if we're not to test on the + * '.arc.gz' suffix. + * + * @return True if this is an Internet Archive GZIP'd ARC file (It begins + * w/ the Internet Archive GZIP header). + * + * @exception IOException If file does not exist or is not unreadable. + */ + public static boolean testCompressedARCFile(File arcFile, + boolean skipSuffixCheck) + throws IOException { + boolean compressedARCFile = false; + isReadable(arcFile); + if(!skipSuffixCheck && !arcFile.getName().toLowerCase() + .endsWith(COMPRESSED_ARC_FILE_EXTENSION)) { + return compressedARCFile; + } + + final InputStream is = new FileInputStream(arcFile); + try { + compressedARCFile = testCompressedARCStream(is); + } finally { + is.close(); + } + return compressedARCFile; + } + + /** + * Tests passed stream is gzip stream by reading in the HEAD. + * Does not reposition the stream. That is left up to the caller. + * @param is An InputStream. + * @return True if compressed stream. + * @throws IOException + */ + public static boolean testCompressedARCStream(final InputStream is) + throws IOException { + boolean compressedARCFile = false; + GzipHeader gh = null; + try { + gh = new GzipHeader(is); + } catch (NoGzipMagicException e ) { + return compressedARCFile; + } + + byte[] fextra = gh.getFextra(); + // Now make sure following bytes are IA GZIP comment. + // First check length. ARC_GZIP_EXTRA_FIELD includes length + // so subtract two and start compare to ARC_GZIP_EXTRA_FIELD + // at +2. + if (fextra != null && + ARC_GZIP_EXTRA_FIELD.length - 2 == fextra.length) { + compressedARCFile = true; + for (int i = 0; i < fextra.length; i++) { + if (fextra[i] != ARC_GZIP_EXTRA_FIELD[i + 2]) { + compressedARCFile = false; + break; + } + } + } + return compressedARCFile; + } + + /** + * Tests passed stream is gzip stream by reading in the HEAD. + * Does reposition of stream when done. + * @param rs An InputStream that is Repositionable. + * @return True if compressed stream. + * @throws IOException + */ + public static boolean testCompressedRepositionalStream( + final RepositionableStream rs) + throws IOException { + boolean compressedARCFile = false; + long p = rs.position(); + try { + compressedARCFile = testCompressedStream((InputStream)rs); + } finally { + rs.position(p); + } + return compressedARCFile; + } + + /** + * Tests passed stream is gzip stream by reading in the HEAD. + * Does reposition of stream when done. + * @param is An InputStream. + * @return True if compressed stream. + * @throws IOException + */ + public static boolean testCompressedStream(final InputStream is) + throws IOException { + boolean compressedARCFile = false; + try { + new GzipHeader(is); + compressedARCFile = true; + } catch (NoGzipMagicException e) { + return compressedARCFile; + } + return compressedARCFile; + } + + /** + * Check file is uncompressed ARC file. + * + * @param arcFile + * File to test if its Internet Archive ARC file uncompressed. + * + * @return True if this is an Internet Archive ARC file. + * + * @exception IOException + * If file does not exist or is not unreadable. + */ + public static boolean testUncompressedARCFile(File arcFile) + throws IOException { + boolean uncompressedARCFile = false; + isReadable(arcFile); + if(arcFile.getName().toLowerCase().endsWith(ARC_FILE_EXTENSION)) { + FileInputStream fis = new FileInputStream(arcFile); + try { + byte [] b = new byte[ARC_MAGIC_NUMBER.length()]; + int read = fis.read(b, 0, ARC_MAGIC_NUMBER.length()); + fis.close(); + if (read == ARC_MAGIC_NUMBER.length()) { + StringBuffer beginStr + = new StringBuffer(ARC_MAGIC_NUMBER.length()); + for (int i = 0; i < ARC_MAGIC_NUMBER.length(); i++) { + beginStr.append((char)b[i]); + } + + if (beginStr.toString(). + equalsIgnoreCase(ARC_MAGIC_NUMBER)) { + uncompressedARCFile = true; + } + } + } finally { + fis.close(); + } + } + + return uncompressedARCFile; + } + + + /** + * @param arcFile File to test. + * @exception IOException If file does not exist or is not unreadable. + */ + private static void isReadable(File arcFile) throws IOException { + if (!arcFile.exists()) { + throw new FileNotFoundException(arcFile.getAbsolutePath() + + " does not exist."); + } + + if (!arcFile.canRead()) { + throw new FileNotFoundException(arcFile.getAbsolutePath() + + " is not readable."); + } + } +} diff --git a/commons/src/main/java/org/archive/io/arc/ARCWriter.java b/commons/src/main/java/org/archive/io/arc/ARCWriter.java new file mode 100644 index 00000000..5c3afb4a --- /dev/null +++ b/commons/src/main/java/org/archive/io/arc/ARCWriter.java @@ -0,0 +1,485 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io.arc; + +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintStream; +import java.io.UnsupportedEncodingException; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.archive.io.GzippedInputStream; +import org.archive.io.ReplayInputStream; +import org.archive.io.WriterPoolMember; +import org.archive.util.ArchiveUtils; +import org.archive.util.DevUtils; +import org.archive.util.MimetypeUtils; + + +/** + * Write ARC files. + * + * Assumption is that the caller is managing access to this ARCWriter ensuring + * only one thread of control accessing this ARC file instance at any one time. + * + *

ARC files are described here: + * Arc + * File Format. This class does version 1 of the ARC file format. It also + * writes version 1.1 which is version 1 with data stuffed into the body of the + * first arc record in the file, the arc file meta record itself. + * + *

An ARC file is three lines of meta data followed by an optional 'body' and + * then a couple of '\n' and then: record, '\n', record, '\n', record, etc. + * If we are writing compressed ARC files, then each of the ARC file records is + * individually gzipped and concatenated together to make up a single ARC file. + * In GZIP terms, each ARC record is a GZIP member of a total gzip'd + * file. + * + *

The GZIPping of the ARC file meta data is exceptional. It is GZIPped + * w/ an extra GZIP header, a special Internet Archive (IA) extra header field + * (e.g. FEXTRA is set in the GZIP header FLG field and an extra field is + * appended to the GZIP header). The extra field has little in it but its + * presence denotes this GZIP as an Internet Archive gzipped ARC. See RFC1952 + * to learn about the GZIP header structure. + * + *

This class then does its GZIPping in the following fashion. Each GZIP + * member is written w/ a new instance of GZIPOutputStream -- actually + * ARCWriterGZIPOututStream so we can get access to the underlying stream. + * The underlying stream stays open across GZIPoutputStream instantiations. + * For the 'special' GZIPing of the ARC file meta data, we cheat by catching the + * GZIPOutputStream output into a byte array, manipulating it adding the + * IA GZIP header, before writing to the stream. + * + *

I tried writing a resettable GZIPOutputStream and could make it work w/ + * the SUN JDK but the IBM JDK threw NPE inside in the deflate.reset -- its zlib + * native call doesn't seem to like the notion of resetting -- so I gave up on + * it. + * + *

Because of such as the above and troubles with GZIPInputStream, we should + * write our own GZIP*Streams, ones that resettable and consious of gzip + * members. + * + *

This class will write until we hit >= maxSize. The check is done at + * record boundary. Records do not span ARC files. We will then close current + * file and open another and then continue writing. + * + *

TESTING: Here is how to test that produced ARC files are good + * using the + * alexa + * ARC c-tools: + * 

+ * % av_procarc hx20040109230030-0.arc.gz | av_ziparc > \
+ *     /tmp/hx20040109230030-0.dat.gz
+ * % av_ripdat /tmp/hx20040109230030-0.dat.gz > /tmp/hx20040109230030-0.cdx
+ *
+ * Examine the produced cdx file to make sure it makes sense. Search + * for 'no-type 0'. If found, then we're opening a gzip record w/o data to + * write. This is bad. + * + *

You can also do gzip -t FILENAME and it will tell you if the + * ARC makes sense to GZIP. + * + *

While being written, ARCs have a '.open' suffix appended. + * + * @author stack + */ +public class ARCWriter extends WriterPoolMember implements ARCConstants { + private static final Logger logger = + Logger.getLogger(ARCWriter.class.getName()); + + /** + * Metadata line pattern. + */ + private static final Pattern METADATA_LINE_PATTERN = + Pattern.compile("^\\S+ \\S+ \\S+ \\S+ \\S+(" + LINE_SEPARATOR + "?)$"); + + private List metadata = null; + + + /** + * Constructor. + * Takes a stream. Use with caution. There is no upperbound check on size. + * Will just keep writing. + * + * @param serialNo used to generate unique file name sequences + * @param out Where to write. + * @param arc File the out is connected to. + * @param cmprs Compress the content written. + * @param metadata File meta data. Can be null. Is list of File and/or + * String objects. + * @param a14DigitDate If null, we'll write current time. + * @throws IOException + */ + public ARCWriter(final AtomicInteger serialNo, final PrintStream out, + final File arc, final boolean cmprs, String a14DigitDate, + final List metadata) + throws IOException { + super(serialNo, out, arc, cmprs, a14DigitDate); + this.metadata = metadata; + writeFirstRecord(a14DigitDate); + } + + /** + * Constructor. + * + * @param serialNo used to generate unique file name sequences + * @param dirs Where to drop the ARC files. + * @param prefix ARC file prefix to use. If null, we use + * DEFAULT_ARC_FILE_PREFIX. + * @param cmprs Compress the ARC files written. The compression is done + * by individually gzipping each record added to the ARC file: i.e. the + * ARC file is a bunch of gzipped records concatenated together. + * @param maxSize Maximum size for ARC files written. + */ + public ARCWriter(final AtomicInteger serialNo, final List dirs, + final String prefix, final boolean cmprs, final long maxSize) { + this(serialNo, dirs, prefix, "", cmprs, maxSize, null); + } + + /** + * Constructor. + * + * @param serialNo used to generate unique file name sequences + * @param dirs Where to drop files. + * @param prefix File prefix to use. + * @param cmprs Compress the records written. + * @param maxSize Maximum size for ARC files written. + * @param suffix File tail to use. If null, unused. + * @param meta File meta data. Can be null. Is list of File and/or + * String objects. + */ + public ARCWriter(final AtomicInteger serialNo, final List dirs, + final String prefix, final String suffix, final boolean cmprs, + final long maxSize, final List meta) { + super(serialNo, dirs, prefix, suffix, cmprs, maxSize, + ARC_FILE_EXTENSION); + this.metadata = meta; + } + + protected String createFile() + throws IOException { + String name = super.createFile(); + writeFirstRecord(getCreateTimestamp()); + return name; + } + + private void writeFirstRecord(final String ts) + throws IOException { + write(generateARCFileMetaData(ts)); + } + + /** + * Write out the ARCMetaData. + * + *

Generate ARC file meta data. Currently we only do version 1 of the + * ARC file formats or version 1.1 when metadata has been supplied (We + * write it into the body of the first record in the arc file). + * + *

Version 1 metadata looks roughly like this: + * + * 

filedesc://testWriteRecord-JunitIAH20040110013326-2.arc 0.0.0.0 \\
+     *  20040110013326 text/plain 77
+     * 1 0 InternetArchive
+     * URL IP-address Archive-date Content-type Archive-length
+     *
+ * + *

If compress is set, then we generate a header that has been gzipped + * in the Internet Archive manner. Such a gzipping enables the FEXTRA + * flag in the FLG field of the gzip header. It then appends an extra + * header field: '8', '0', 'L', 'X', '0', '0', '0', '0'. The first two + * bytes are the length of the field and the last 6 bytes the Internet + * Archive header. To learn about GZIP format, see RFC1952. To learn + * about the Internet Archive extra header field, read the source for + * av_ziparc which can be found at + * alexa/vista/alexa-tools-1.2/src/av_ziparc.cc. + * + *

We do things in this roundabout manner because the java + * GZIPOutputStream does not give access to GZIP header fields. + * + * @param date Date to put into the ARC metadata. + * + * @return Byte array filled w/ the arc header. + * @throws IOException + */ + private byte [] generateARCFileMetaData(String date) + throws IOException { + int metadataBodyLength = getMetadataLength(); + // If metadata body, then the minor part of the version is '1' rather + // than '0'. + String metadataHeaderLinesTwoAndThree = + getMetadataHeaderLinesTwoAndThree("1 " + + ((metadataBodyLength > 0)? "1": "0")); + int recordLength = metadataBodyLength + + metadataHeaderLinesTwoAndThree.getBytes(DEFAULT_ENCODING).length; + String metadataHeaderStr = ARC_MAGIC_NUMBER + getBaseFilename() + + " 0.0.0.0 " + date + " text/plain " + recordLength + + metadataHeaderLinesTwoAndThree; + ByteArrayOutputStream metabaos = + new ByteArrayOutputStream(recordLength); + // Write the metadata header. + metabaos.write(metadataHeaderStr.getBytes(DEFAULT_ENCODING)); + // Write the metadata body, if anything to write. + if (metadataBodyLength > 0) { + writeMetaData(metabaos); + } + + // Write out a LINE_SEPARATORs to end this record. + metabaos.write(LINE_SEPARATOR); + + // Now get bytes of all just written and compress if flag set. + byte [] bytes = metabaos.toByteArray(); + + if(isCompressed()) { + // GZIP the header but catch the gzipping into a byte array so we + // can add the special IA GZIP header to the product. After + // manipulations, write to the output stream (The JAVA GZIP + // implementation does not give access to GZIP header. It + // produces a 'default' header only). We can get away w/ these + // maniupulations because the GZIP 'default' header doesn't + // do the 'optional' CRC'ing of the header. + byte [] gzippedMetaData = GzippedInputStream.gzip(bytes); + if (gzippedMetaData[3] != 0) { + throw new IOException("The GZIP FLG header is unexpectedly " + + " non-zero. Need to add smarter code that can deal " + + " when already extant extra GZIP header fields."); + } + // Set the GZIP FLG header to '4' which says that the GZIP header + // has extra fields. Then insert the alex {'L', 'X', '0', '0', '0, + // '0'} 'extra' field. The IA GZIP header will also set byte + // 9 (zero-based), the OS byte, to 3 (Unix). We'll do the same. + gzippedMetaData[3] = 4; + gzippedMetaData[9] = 3; + byte [] assemblyBuffer = new byte[gzippedMetaData.length + + ARC_GZIP_EXTRA_FIELD.length]; + // '10' in the below is a pointer past the following bytes of the + // GZIP header: ID1 ID2 CM FLG + MTIME(4-bytes) XFL OS. See + // RFC1952 for explaination of the abbreviations just used. + System.arraycopy(gzippedMetaData, 0, assemblyBuffer, 0, 10); + System.arraycopy(ARC_GZIP_EXTRA_FIELD, 0, assemblyBuffer, 10, + ARC_GZIP_EXTRA_FIELD.length); + System.arraycopy(gzippedMetaData, 10, assemblyBuffer, + 10 + ARC_GZIP_EXTRA_FIELD.length, gzippedMetaData.length - 10); + bytes = assemblyBuffer; + } + return bytes; + } + + public String getMetadataHeaderLinesTwoAndThree(String version) { + StringBuffer buffer = new StringBuffer(); + buffer.append(LINE_SEPARATOR); + buffer.append(version); + buffer.append(" InternetArchive"); + buffer.append(LINE_SEPARATOR); + buffer.append("URL IP-address Archive-date Content-type Archive-length"); + buffer.append(LINE_SEPARATOR); + return buffer.toString(); + } + + /** + * Write all metadata to passed baos. + * + * @param baos Byte array to write to. + * @throws UnsupportedEncodingException + * @throws IOException + */ + private void writeMetaData(ByteArrayOutputStream baos) + throws UnsupportedEncodingException, IOException { + if (this.metadata == null) { + return; + } + + for (Iterator i = this.metadata.iterator(); + i.hasNext();) { + Object obj = i.next(); + if (obj instanceof String) { + baos.write(((String)obj).getBytes(DEFAULT_ENCODING)); + } else if (obj instanceof File) { + InputStream is = null; + try { + is = new BufferedInputStream( + new FileInputStream((File)obj)); + byte [] buffer = new byte[4096]; + for (int read = -1; (read = is.read(buffer)) != -1;) { + baos.write(buffer, 0, read); + } + } finally { + if (is != null) { + is.close(); + } + } + } else if (obj != null) { + logger.severe("Unsupported metadata type: " + obj); + } + } + return; + } + + /** + * @return Total length of metadata. + * @throws UnsupportedEncodingException + */ + private int getMetadataLength() + throws UnsupportedEncodingException { + int result = -1; + if (this.metadata == null) { + result = 0; + } else { + for (Iterator i = this.metadata.iterator(); + i.hasNext();) { + Object obj = i.next(); + if (obj instanceof String) { + result += ((String)obj).getBytes(DEFAULT_ENCODING).length; + } else if (obj instanceof File) { + result += ((File)obj).length(); + } else { + logger.severe("Unsupported metadata type: " + obj); + } + } + } + return result; + } + + /** + * @deprecated use input-stream version directly instead + */ + public void write(String uri, String contentType, String hostIP, + long fetchBeginTimeStamp, long recordLength, + ByteArrayOutputStream baos) + throws IOException { + write(uri, contentType, hostIP, fetchBeginTimeStamp, recordLength, + new ByteArrayInputStream(baos.toByteArray()), false); + } + + public void write(String uri, String contentType, String hostIP, + long fetchBeginTimeStamp, long recordLength, InputStream in) + throws IOException { + write(uri,contentType,hostIP,fetchBeginTimeStamp,recordLength,in,true); + } + + /** + * Write a record with the given metadata/content. + * + * @param uri + * URI for metadata-line + * @param contentType + * MIME content-type for metadata-line + * @param hostIP + * IP for metadata-line + * @param fetchBeginTimeStamp + * timestamp for metadata-line + * @param recordLength + * length for metadata-line; also may be enforced + * @param in + * source InputStream for record content + * @param enforceLength + * whether to enforce the declared length; should be true + * unless intentionally writing bad records for testing + * @throws IOException + */ + public void write(String uri, String contentType, String hostIP, + long fetchBeginTimeStamp, long recordLength, InputStream in, + boolean enforceLength) throws IOException { + preWriteRecordTasks(); + try { + write(getMetaLine(uri, contentType, hostIP, fetchBeginTimeStamp, + recordLength).getBytes(UTF8)); + copyFrom(in, recordLength, enforceLength); + if (in instanceof ReplayInputStream) { + // check for consumption of entire recorded material + long remaining = ((ReplayInputStream) in).remaining(); + // Should be zero at this stage. If not, something is + // wrong. + if (remaining != 0) { + String message = "Gap between expected and actual: " + + remaining + LINE_SEPARATOR + DevUtils.extraInfo() + + " writing arc " + + this.getFile().getAbsolutePath(); + DevUtils.warnHandle(new Throwable(message), message); + throw new IOException(message); + } + } + write(LINE_SEPARATOR); + } finally { + postWriteRecordTasks(); + } + } + + /** + * @param uri + * @param contentType + * @param hostIP + * @param fetchBeginTimeStamp + * @param recordLength + * @return Metadata line for an ARCRecord made of passed components. + * @exception IOException + */ + protected String getMetaLine(String uri, String contentType, String hostIP, + long fetchBeginTimeStamp, long recordLength) + throws IOException { + if (fetchBeginTimeStamp <= 0) { + throw new IOException("Bogus fetchBeginTimestamp: " + + Long.toString(fetchBeginTimeStamp)); + } + + return validateMetaLine(createMetaline(uri, hostIP, + ArchiveUtils.get14DigitDate(fetchBeginTimeStamp), + MimetypeUtils.truncate(contentType), + Long.toString(recordLength))); + } + + public String createMetaline(String uri, String hostIP, + String timeStamp, String mimetype, String recordLength) { + return uri + HEADER_FIELD_SEPARATOR + hostIP + + HEADER_FIELD_SEPARATOR + timeStamp + + HEADER_FIELD_SEPARATOR + mimetype + + HEADER_FIELD_SEPARATOR + recordLength + LINE_SEPARATOR; + } + + /** + * Test that the metadata line is valid before writing. + * @param metaLineStr + * @throws IOException + * @return The passed in metaline. + */ + protected String validateMetaLine(String metaLineStr) + throws IOException { + if (metaLineStr.length() > MAX_METADATA_LINE_LENGTH) { + throw new IOException("Metadata line length is " + + metaLineStr.length() + " which is > than maximum " + + MAX_METADATA_LINE_LENGTH); + } + Matcher m = METADATA_LINE_PATTERN.matcher(metaLineStr); + if (!m.matches()) { + throw new IOException("Metadata line doesn't match expected" + + " pattern: " + metaLineStr); + } + return metaLineStr; + } +} diff --git a/commons/src/main/java/org/archive/io/arc/ARCWriterPool.java b/commons/src/main/java/org/archive/io/arc/ARCWriterPool.java new file mode 100644 index 00000000..5463a253 --- /dev/null +++ b/commons/src/main/java/org/archive/io/arc/ARCWriterPool.java @@ -0,0 +1,79 @@ +/* ARCWriterPool + * + * $Id$ + * + * Created on Jan 22, 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.io.arc; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.pool.BasePoolableObjectFactory; +import org.archive.io.WriterPool; +import org.archive.io.WriterPoolMember; +import org.archive.io.WriterPoolSettings; + + +/** + * A pool of ARCWriters. + * + * @author stack + */ +public class ARCWriterPool extends WriterPool { + /** + * Constructor + * + * @param settings Settings for this pool. + * @param poolMaximumActive + * @param poolMaximumWait + */ + public ARCWriterPool(final WriterPoolSettings settings, + final int poolMaximumActive, final int poolMaximumWait) { + this(new AtomicInteger(), settings, poolMaximumActive, poolMaximumWait); + } + + /** + * Constructor + * + * @param serial Used to generate unique filename sequences + * @param settings Settings for this pool. + * @param poolMaximumActive + * @param poolMaximumWait + */ + public ARCWriterPool(final AtomicInteger serial, + final WriterPoolSettings settings, + final int poolMaximumActive, final int poolMaximumWait) { + super(serial, new BasePoolableObjectFactory() { + public Object makeObject() throws Exception { + return new ARCWriter(serial, settings.getOutputDirs(), + settings.getPrefix(), settings.getSuffix(), + settings.isCompressed(), settings.getMaxSize(), + settings.getMetadata()); + } + + public void destroyObject(Object arcWriter) + throws Exception { + ((WriterPoolMember)arcWriter).close(); + super.destroyObject(arcWriter); + } + }, settings, poolMaximumActive, poolMaximumWait); + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/io/arc/package.html b/commons/src/main/java/org/archive/io/arc/package.html new file mode 100644 index 00000000..d1798b80 --- /dev/null +++ b/commons/src/main/java/org/archive/io/arc/package.html @@ -0,0 +1,9 @@ + + + +org.archive.io.arc package + + +ARC file reading and writing. + + diff --git a/commons/src/main/java/org/archive/io/warc/WARCConstants.java b/commons/src/main/java/org/archive/io/warc/WARCConstants.java new file mode 100644 index 00000000..0617c8df --- /dev/null +++ b/commons/src/main/java/org/archive/io/warc/WARCConstants.java @@ -0,0 +1,233 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io.warc; + +import java.util.Arrays; +import java.util.List; + +import org.archive.io.ArchiveFileConstants; + +/** + * WARC Constants used by WARC readers and writers. + * Below constants are used by version 0.10 and 0.12 of WARC Reader/Writer. + * @author stack + * @version $Revision$ $Date$ + */ +public interface WARCConstants extends ArchiveFileConstants { + /** + * Default maximum WARC file size. + * 1Gig. + */ + public static final int DEFAULT_MAX_WARC_FILE_SIZE = 1024 * 1024 * 1024; + + /** + * WARC MAGIC + * WARC files and records begin with this sequence. + */ + public static final String WARC_MAGIC = "WARC/"; + + /** + * Hard-coded version for WARC files made with this code. + * Setting to 0.10 because differs from 0.9 spec. See accompanying + * package documentation. + */ + public static final String WARC_VERSION = "0.17"; + + /** + * Assumed maximum size of a Header Line. + * + * This 100k which seems massive but its the same as the LINE_LENGTH from + * alexa/include/a_arcio.h: + * 

+     * #define LINE_LENGTH     (100*1024)
+     *
+ */ + public static final int MAX_WARC_HEADER_LINE_LENGTH = 1024 * 100; + public static final int MAX_LINE_LENGTH = MAX_WARC_HEADER_LINE_LENGTH; + + /** + * WARC file extention. + */ + public static final String WARC_FILE_EXTENSION = "warc"; + + /** + * Dot WARC file extension. + */ + public static final String DOT_WARC_FILE_EXTENSION = + "." + WARC_FILE_EXTENSION; + + public static final String DOT_COMPRESSED_FILE_EXTENSION = + ArchiveFileConstants.DOT_COMPRESSED_FILE_EXTENSION; + + /** + * Compressed WARC file extension. + */ + public static final String COMPRESSED_WARC_FILE_EXTENSION = + WARC_FILE_EXTENSION + DOT_COMPRESSED_FILE_EXTENSION; + + /** + * Compressed dot WARC file extension. + */ + public static final String DOT_COMPRESSED_WARC_FILE_EXTENSION = + DOT_WARC_FILE_EXTENSION + DOT_COMPRESSED_FILE_EXTENSION; + + /** + * Encoding to use getting bytes from strings. + * + * Specify an encoding rather than leave it to chance: i.e whatever the + * JVMs encoding. Use an encoding that gets the stream as bytes, not chars. + * + *

TODO: ARC uses ISO-8859-1. In general, we should use UTF-8 but we + * probably need a single byte encoding if we're out for preserving the + * binary data as received over the net (We probably don't want to transform + * the supra-ASCII characters to UTF-8 before storing in ARC). For now, + * till we figure it, DEFAULT_ENCODING is single-byte charset -- same as + * ARCs. + */ + public static final String DEFAULT_ENCODING = "UTF-8"; + public static final String HEADER_LINE_ENCODING = DEFAULT_ENCODING; + + // TODO: Revisit. 8859 isn't correct, especially if we settle on RFC822 + // headers + public static final String WARC_HEADER_ENCODING = HEADER_LINE_ENCODING; + + public static final String [] HEADER_FIELD_KEYS = { + VERSION_FIELD_KEY, + LENGTH_FIELD_KEY, + TYPE_FIELD_KEY, + URL_FIELD_KEY, + DATE_FIELD_KEY, + RECORD_IDENTIFIER_FIELD_KEY, + MIMETYPE_FIELD_KEY + }; + + /** + * WARC Record Types. + */ + public static final String WARCINFO = "warcinfo"; + public static final String RESPONSE = "response"; + public static final String RESOURCE = "resource"; + public static final String REQUEST = "request"; + public static final String METADATA = "metadata"; + public static final String REVISIT = "revisit"; + public static final String CONVERSION = "conversion"; + public static final String CONTINUATION = "continuation"; + + public static final String TYPE = "type"; + + // List of all WARC Record TYPES + public static final String [] TYPES = {WARCINFO, RESPONSE, RESOURCE, + REQUEST, METADATA, REVISIT, CONVERSION, CONTINUATION}; + + // Indices into TYPES array. + public static final int WARCINFO_INDEX = 0; + public static final int RESPONSE_INDEX = 1; + public static final int RESOURCE_INDEX = 2; + public static final int REQUEST_INDEX = 3; + public static final int METADATA_INDEX = 4; + public static final int REVISIT_INDEX = 5; + public static final int CONVERSION_INDEX = 6; + public static final int CONTINUATION_INDEX = 7; + + // TYPES as List. + public static final List TYPES_LIST = Arrays.asList(TYPES); + + /** + * WARC-ID + */ + public static final String WARC_ID = WARC_MAGIC + WARC_VERSION; + + /** + * Header field seperator character. + */ + public static final char HEADER_FIELD_SEPARATOR = ' '; + + /** + * WSP + * One of a space or horizontal tab character. + * TODO: WSP undefined. Fix. + */ + public static final Character [] WSP = {HEADER_FIELD_SEPARATOR, '\t'}; + + /** + * Placeholder for length in Header line. + * Placeholder is same size as the fixed field size allocated for length, + * 12 characters. 12 characters allows records of size almost 1TB. + */ + public static final String PLACEHOLDER_RECORD_LENGTH_STRING = + "000000000000"; + + public static final String NAMED_FIELD_IP_LABEL = "IP-Address"; + public static final String NAMED_FIELD_CHECKSUM_LABEL = "Checksum"; + public static final String NAMED_FIELD_RELATED_LABEL = "References"; + public static final String NAMED_FIELD_WARCFILENAME = "Filename"; + public static final String NAMED_FIELD_DESCRIPTION = "Description"; + public static final String NAMED_FIELD_FILEDESC = "ARC-FileDesc"; + public static final String NAMED_FIELD_TRUNCATED = "Truncated"; + public static final String NAMED_FIELD_TRUNCATED_VALUE_TIME = "time"; + public static final String NAMED_FIELD_TRUNCATED_VALUE_LENGTH = "length"; + + public static final String NAMED_FIELD_TRUNCATED_VALUE_HEAD = + "long-headers"; + public static final String NAMED_FIELD_TRUNCATED_VALUE_UNSPECIFIED = null; + + // Headers for version 0.17 of spec. + public static final String HEADER_KEY_URI = "WARC-Target-URI"; + public static final String HEADER_KEY_DATE = "WARC-Date"; + public static final String HEADER_KEY_TYPE = "WARC-Type"; + public static final String HEADER_KEY_ID = "WARC-Record-ID"; + public static final String HEADER_KEY_IP = "WARC-IP-Address"; + public static final String HEADER_KEY_BLOCK_DIGEST = "WARC-Block-Digest"; + public static final String HEADER_KEY_PAYLOAD_DIGEST = "WARC-Payload-Digest"; + public static final String HEADER_KEY_CONCURRENT_TO = + "WARC-Concurrent-To"; + public static final String HEADER_KEY_TRUNCATED = "WARC-Truncated"; + public static final String HEADER_KEY_PROFILE = "WARC-Profile"; + public static final String HEADER_KEY_FILENAME = "WARC-Filename"; + public static final String HEADER_KEY_ETAG = "WARC-Etag"; + public static final String HEADER_KEY_LAST_MODIFIED = "WARC-Last-Modified"; + + public static final String PROFILE_REVISIT_IDENTICAL_DIGEST = + "http://netpreserve.org/warc/0.17/revisit/identical-payload-digest"; + public static final String PROFILE_REVISIT_NOT_MODIFIED = + "http://netpreserve.org/warc/0.17/revisit/server-not-modified"; + + public static final String CONTENT_LENGTH = "Content-Length"; + public static final String CONTENT_TYPE = "Content-Type"; + public static final String CONTENT_DESCRIPTION = "Content-Description"; + + public static final String COLON_SPACE = ": "; + + public static final String TRUNCATED_VALUE_UNSPECIFIED = "unspecified"; + + + /** + * To be safe, lets use application type rather than message. Regards + * 'message/http', RFC says "...provided that it obeys the MIME restrictions + * for all 'message' types regarding line length and encodings." This + * usually means lines of 1000 octets max (unless a + * 'Content-Transfer-Encoding: binary' mime header is present). + * @see rfc2616 section 19.1 + */ + public static final String HTTP_REQUEST_MIMETYPE = + "application/http; msgtype=request"; + public static final String HTTP_RESPONSE_MIMETYPE = + "application/http; msgtype=response"; +} diff --git a/commons/src/main/java/org/archive/io/warc/WARCReader.java b/commons/src/main/java/org/archive/io/warc/WARCReader.java new file mode 100644 index 00000000..74de4929 --- /dev/null +++ b/commons/src/main/java/org/archive/io/warc/WARCReader.java @@ -0,0 +1,288 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io.warc; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.Iterator; +import java.util.List; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; +import org.apache.commons.lang.NotImplementedException; +import org.archive.io.ArchiveReader; +import org.archive.io.ArchiveRecord; +import org.archive.io.warc.WARCConstants; + +/** + * WARCReader. + * Go via {@link WARCReaderFactory} to get instance. + * @author stack + * @version $Date: 2006-11-27 18:03:03 -0800 (Mon, 27 Nov 2006) $ $Version$ + */ +public class WARCReader extends ArchiveReader implements WARCConstants { + WARCReader() { + super(); + } + + @Override + protected void initialize(String i) { + super.initialize(i); + setVersion(WARC_VERSION); + } + + /** + * Skip over any trailing new lines at end of the record so we're lined up + * ready to read the next. + * @param record + * @throws IOException + */ + protected void gotoEOR(ArchiveRecord record) throws IOException { + if (record.available() != 0) { + throw new IOException("Record should be exhausted before coming " + + "in here"); + } + + // Records end in 2*CRLF. Suck it up. + readExpectedChar(getIn(), CRLF.charAt(0)); + readExpectedChar(getIn(), CRLF.charAt(1)); + readExpectedChar(getIn(), CRLF.charAt(0)); + readExpectedChar(getIn(), CRLF.charAt(1)); + } + + protected void readExpectedChar(final InputStream is, final int expected) + throws IOException { + int c = is.read(); + if (c != expected) { + throw new IOException("Unexpected character " + + Integer.toHexString(c) + "(Expecting " + + Integer.toHexString(expected) + ")"); + } + } + + /** + * Create new WARC record. + * Encapsulate housekeeping that has to do w/ creating new Record. + * @param is InputStream to use. + * @param offset Absolute offset into WARC file. + * @return A WARCRecord. + * @throws IOException + */ + protected WARCRecord createArchiveRecord(InputStream is, long offset) + throws IOException { + return (WARCRecord)currentRecord(new WARCRecord(is, + getReaderIdentifier(), offset, isDigest(), isStrict())); + } + + @Override + public void dump(boolean compress) + throws IOException, java.text.ParseException { + for (final Iterator i = iterator(); i.hasNext();) { + ArchiveRecord r = i.next(); + System.out.println(r.getHeader().toString()); + r.dump(); + System.out.println(); + } + } + + + @Override + public ArchiveReader getDeleteFileOnCloseReader(final File f) { + throw new NotImplementedException("TODO"); + } + + @Override + public String getDotFileExtension() { + return DOT_WARC_FILE_EXTENSION; + } + + @Override + public String getFileExtension() { + return WARC_FILE_EXTENSION; + } + + // Static methods follow. Mostly for command-line processing. + + /** + * + * @param formatter Help formatter instance. + * @param options Usage options. + * @param exitCode Exit code. + */ + private static void usage(HelpFormatter formatter, Options options, + int exitCode) { + formatter.printHelp("java org.archive.io.arc.WARCReader" + + " [--digest=true|false] \\\n" + + " [--format=cdx|cdxfile|dump|gzipdump]" + + " [--offset=#] \\\n[--strict] [--parse] WARC_FILE|WARC_URL", + options); + System.exit(exitCode); + } + + /** + * Write out the arcfile. + * + * @param reader + * @param format Format to use outputting. + * @throws IOException + * @throws java.text.ParseException + */ + protected static void output(WARCReader reader, String format) + throws IOException, java.text.ParseException { + if (!reader.output(format)) { + throw new IOException("Unsupported format: " + format); + } + } + + /** + * Generate a CDX index file for an ARC file. + * + * @param urlOrPath The ARC file to generate a CDX index for + * @throws IOException + * @throws java.text.ParseException + */ + public static void createCDXIndexFile(String urlOrPath) + throws IOException, java.text.ParseException { + WARCReader r = WARCReaderFactory.get(urlOrPath); + r.setStrict(false); + r.setDigest(true); + output(r, CDX_FILE); + } + + /** + * Command-line interface to WARCReader. + * + * Here is the command-line interface: + * 

+     * usage: java org.archive.io.arc.WARCReader [--offset=#] ARCFILE
+     *  -h,--help      Prints this message and exits.
+     *  -o,--offset    Outputs record at this offset into arc file.
+ * + *

Outputs using a pseudo-CDX format as described here: + * CDX + * Legent and here + * Example. + * Legend used in below is: 'CDX b e a m s c V (or v if uncompressed) n g'. + * Hash is hard-coded straight SHA-1 hash of content. + * + * @param args Command-line arguments. + * @throws ParseException Failed parse of the command line. + * @throws IOException + * @throws java.text.ParseException + */ + @SuppressWarnings("unchecked") + public static void main(String [] args) + throws ParseException, IOException, java.text.ParseException { + Options options = getOptions(); + PosixParser parser = new PosixParser(); + CommandLine cmdline = parser.parse(options, args, false); + List cmdlineArgs = cmdline.getArgList(); + Option [] cmdlineOptions = cmdline.getOptions(); + HelpFormatter formatter = new HelpFormatter(); + + // If no args, print help. + if (cmdlineArgs.size() <= 0) { + usage(formatter, options, 0); + } + + // Now look at options passed. + long offset = -1; + boolean digest = false; + boolean strict = false; + String format = CDX; + for (int i = 0; i < cmdlineOptions.length; i++) { + switch(cmdlineOptions[i].getId()) { + case 'h': + usage(formatter, options, 0); + break; + + case 'o': + offset = + Long.parseLong(cmdlineOptions[i].getValue()); + break; + + case 's': + strict = true; + break; + + case 'd': + digest = getTrueOrFalse(cmdlineOptions[i].getValue()); + break; + + case 'f': + format = cmdlineOptions[i].getValue().toLowerCase(); + boolean match = false; + // List of supported formats. + final String [] supportedFormats = + {CDX, DUMP, GZIP_DUMP, CDX_FILE}; + for (int ii = 0; ii < supportedFormats.length; ii++) { + if (supportedFormats[ii].equals(format)) { + match = true; + break; + } + } + if (!match) { + usage(formatter, options, 1); + } + break; + + default: + throw new RuntimeException("Unexpected option: " + + + cmdlineOptions[i].getId()); + } + } + + if (offset >= 0) { + if (cmdlineArgs.size() != 1) { + System.out.println("Error: Pass one arcfile only."); + usage(formatter, options, 1); + } + WARCReader r = WARCReaderFactory.get( + new File((String)cmdlineArgs.get(0)), offset); + r.setStrict(strict); + outputRecord(r, format); + } else { + for (Iterator i = cmdlineArgs.iterator(); i.hasNext();) { + String urlOrPath = (String)i.next(); + try { + WARCReader r = WARCReaderFactory.get(urlOrPath); + r.setStrict(strict); + r.setDigest(digest); + output(r, format); + } catch (RuntimeException e) { + // Write out name of file we failed on to help with + // debugging. Then print stack trace and try to keep + // going. We do this for case where we're being fed + // a bunch of ARCs; just note the bad one and move + // on to the next. + System.err.println("Exception processing " + urlOrPath + + ": " + e.getMessage()); + e.printStackTrace(System.err); + System.exit(1); + } + } + } + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/io/warc/WARCReaderFactory.java b/commons/src/main/java/org/archive/io/warc/WARCReaderFactory.java new file mode 100644 index 00000000..c538f763 --- /dev/null +++ b/commons/src/main/java/org/archive/io/warc/WARCReaderFactory.java @@ -0,0 +1,296 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io.warc; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Iterator; + +import org.archive.io.ArchiveReader; +import org.archive.io.ArchiveReaderFactory; +import org.archive.io.ArchiveRecord; +import org.archive.io.GzippedInputStream; +import org.archive.io.warc.WARCConstants; +import org.archive.util.FileUtils; + +/** + * Factory for WARC Readers. + * Figures whether to give out a compressed file Reader or an uncompressed + * Reader. + * @author stack + * @version $Date: 2006-08-23 17:59:04 -0700 (Wed, 23 Aug 2006) $ $Version$ + */ +public class WARCReaderFactory extends ArchiveReaderFactory +implements WARCConstants { + private static final WARCReaderFactory factory = new WARCReaderFactory(); + + /** + * Shutdown any access to default constructor. + * This factory is Singleton. + */ + private WARCReaderFactory() { + super(); + } + + public static WARCReader get(String arcFileOrUrl) + throws MalformedURLException, IOException { + return (WARCReader)WARCReaderFactory.factory. + getArchiveReader(arcFileOrUrl); + } + + public static WARCReader get(final File f) throws IOException { + return (WARCReader)WARCReaderFactory.factory.getArchiveReader(f); + } + + /** + * @param f An arcfile to read. + * @param offset Have returned Reader set to start reading at this offset. + * @return A WARCReader. + * @throws IOException + */ + public static WARCReader get(final File f, final long offset) + throws IOException { + return (WARCReader)WARCReaderFactory.factory. + getArchiveReader(f, offset); + } + + protected ArchiveReader getArchiveReader(final File f, final long offset) + throws IOException { + boolean compressed = testCompressedWARCFile(f); + if (!compressed) { + if (!FileUtils.isReadableWithExtensionAndMagic(f, + DOT_WARC_FILE_EXTENSION, WARC_MAGIC)) { + throw new IOException(f.getAbsolutePath() + + " is not a WARC file."); + } + } + return (WARCReader)(compressed? + WARCReaderFactory.factory.new CompressedWARCReader(f, offset): + WARCReaderFactory.factory.new UncompressedWARCReader(f, offset)); + } + + public static ArchiveReader get(final String s, final InputStream is, + final boolean atFirstRecord) + throws IOException { + return WARCReaderFactory.factory.getArchiveReader(s, is, + atFirstRecord); + } + + protected ArchiveReader getArchiveReader(final String f, + final InputStream is, final boolean atFirstRecord) + throws IOException { + // For now, assume stream is compressed. Later add test of input + // stream or handle exception thrown when figure not compressed stream. + return new CompressedWARCReader(f, asRepositionable(is), atFirstRecord); + } + + public static WARCReader get(final URL arcUrl, final long offset) + throws IOException { + return (WARCReader)WARCReaderFactory.factory.getArchiveReader(arcUrl, + offset); + } + + /** + * Get an ARCReader. + * Pulls the ARC local into whereever the System Property + * java.io.tmpdir points. It then hands back an ARCReader that + * points at this local copy. A close on this ARCReader instance will + * remove the local copy. + * @param arcUrl An URL that points at an ARC. + * @return An ARCReader. + * @throws IOException + */ + public static WARCReader get(final URL arcUrl) + throws IOException { + return (WARCReader)WARCReaderFactory.factory.getArchiveReader(arcUrl); + } + + /** + * Check file is compressed WARC. + * + * @param f File to test. + * + * @return True if this is compressed WARC (TODO: Just tests if file is + * GZIP'd file (It begins w/ GZIP MAGIC)). + * + * @exception IOException If file does not exist or is not unreadable. + */ + public static boolean testCompressedWARCFile(final File f) + throws IOException { + FileUtils.assertReadable(f); + boolean compressed = false; + final InputStream is = new FileInputStream(f); + try { + compressed = GzippedInputStream.isCompressedStream(is); + } finally { + is.close(); + } + return compressed; + } + + /** + * Uncompressed WARC file reader. + * @author stack + */ + private class UncompressedWARCReader extends WARCReader { + /** + * Constructor. + * @param f Uncompressed arcfile to read. + * @throws IOException + */ + public UncompressedWARCReader(final File f) + throws IOException { + this(f, 0); + } + + /** + * Constructor. + * + * @param f Uncompressed file to read. + * @param offset Offset at which to position Reader. + * @throws IOException + */ + public UncompressedWARCReader(final File f, final long offset) + throws IOException { + // File has been tested for existence by time it has come to here. + setIn(getInputStream(f, offset)); + initialize(f.getAbsolutePath()); + } + + /** + * Constructor. + * + * @param f Uncompressed file to read. + * @param is InputStream. + */ + public UncompressedWARCReader(final String f, final InputStream is) { + // Arc file has been tested for existence by time it has come + // to here. + setIn(is); + initialize(f); + } + } + + /** + * Compressed WARC file reader. + * + * @author stack + */ + private class CompressedWARCReader extends WARCReader { + /** + * Constructor. + * + * @param f Compressed file to read. + * @throws IOException + */ + public CompressedWARCReader(final File f) throws IOException { + this(f, 0); + } + + /** + * Constructor. + * + * @param f Compressed arcfile to read. + * @param offset Position at where to start reading file. + * @throws IOException + */ + public CompressedWARCReader(final File f, final long offset) + throws IOException { + // File has been tested for existence by time it has come to here. + setIn(new GzippedInputStream(getInputStream(f, offset))); + setCompressed((offset == 0)); + initialize(f.getAbsolutePath()); + } + + /** + * Constructor. + * + * @param f Compressed arcfile. + * @param is InputStream to use. + * @param atFirstRecord + * @throws IOException + */ + public CompressedWARCReader(final String f, final InputStream is, + final boolean atFirstRecord) + throws IOException { + // Arc file has been tested for existence by time it has come + // to here. + setIn(new GzippedInputStream(is)); + setCompressed(true); + initialize(f); + // TODO: Ignore atFirstRecord. Probably doesn't apply in WARC world. + } + + /** + * Get record at passed offset. + * + * @param offset Byte index into file at which a record starts. + * @return A WARCRecord reference. + * @throws IOException + */ + public WARCRecord get(long offset) throws IOException { + cleanupCurrentRecord(); + ((GzippedInputStream)getIn()).gzipMemberSeek(offset); + return (WARCRecord) createArchiveRecord(getIn(), offset); + } + + public Iterator iterator() { + /** + * Override ArchiveRecordIterator so can base returned iterator on + * GzippedInputStream iterator. + */ + return new ArchiveRecordIterator() { + private GzippedInputStream gis = + (GzippedInputStream)getInputStream(); + + private Iterator gzipIterator = this.gis.iterator(); + + protected boolean innerHasNext() { + return this.gzipIterator.hasNext(); + } + + protected ArchiveRecord innerNext() throws IOException { + // Get the positoin before gzipIterator.next moves + // it on past the gzip header. + long p = this.gis.position(); + InputStream is = (InputStream) this.gzipIterator.next(); + return createArchiveRecord(is, p); + } + }; + } + + protected void gotoEOR(ArchiveRecord rec) throws IOException { + // TODO + } + } + + public static boolean isWARCSuffix(final String f) { + return (f == null)? + false: + (f.toLowerCase().endsWith(DOT_COMPRESSED_WARC_FILE_EXTENSION))? + true: + (f.toLowerCase().endsWith(DOT_WARC_FILE_EXTENSION))? + true: false; + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/io/warc/WARCRecord.java b/commons/src/main/java/org/archive/io/warc/WARCRecord.java new file mode 100644 index 00000000..9ef134af --- /dev/null +++ b/commons/src/main/java/org/archive/io/warc/WARCRecord.java @@ -0,0 +1,225 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io.warc; + +import it.unimi.dsi.fastutil.io.RepositionableStream; + +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.httpclient.Header; +import org.apache.commons.httpclient.HttpParser; +import org.archive.io.ArchiveRecord; +import org.archive.io.ArchiveRecordHeader; + + +/** + * A WARC file Record. + * + * @author stack + */ +public class WARCRecord extends ArchiveRecord implements WARCConstants { + private Pattern WHITESPACE = Pattern.compile("\\s"); + + /** + * Constructor. + * + * @param in Stream cue'd up to be at the start of the record this instance + * is to represent. + * @throws IOException + */ + public WARCRecord(InputStream in, final String identifier, + final long offset) + throws IOException { + this(in, identifier, offset, true, false); + } + + /** + * Constructor. + * @param in Stream cue'd up just past Header Line and Named Fields. + * @param headers Header Line and ANVL Named fields. + * @throws IOException + */ + public WARCRecord(InputStream in, ArchiveRecordHeader headers) + throws IOException { + super(in, headers, 0, true, false); + } + + /** + * Constructor. + * + * @param in Stream cue'd up to be at the start of the record this instance + * is to represent or, if headers is not null, just past the + * Header Line and Named Fields. + * @param identifier Identifier for this the hosting Reader. + * @param offset Current offset into in (Used to keep + * position properly aligned). Usually 0. + * @param digest True if we're to calculate digest for this record. Not + * digesting saves about ~15% of cpu during parse. + * @param strict Be strict parsing (Parsing stops if file inproperly + * formatted). + * @throws IOException + */ + public WARCRecord(final InputStream in, final String identifier, + final long offset, boolean digest, boolean strict) + throws IOException { + super(in, null, 0, digest, strict); + setHeader(parseHeaders(in, identifier, offset, strict)); + } + + /** + * Parse WARC Header Line and Named Fields. + * @param in Stream to read. + * @param identifier Identifier for the hosting Reader. + * @param offset Absolute offset into Reader. + * @param strict Whether to be loose parsing or not. + * @return An ArchiveRecordHeader. + * @throws IOException + */ + protected ArchiveRecordHeader parseHeaders(final InputStream in, + final String identifier, final long offset, final boolean strict) + throws IOException { + final Map m = new HashMap(); + m.put(ABSOLUTE_OFFSET_KEY, new Long(offset)); + m.put(READER_IDENTIFIER_FIELD_KEY, identifier); + + long startPosition = -1; + if (in instanceof RepositionableStream) { + startPosition = ((RepositionableStream)in).position(); + } + String firstLine = + new String(HttpParser.readLine(in, WARC_HEADER_ENCODING)); + if (firstLine == null || firstLine.length() <=0) { + throw new IOException("Failed to read WARC_MAGIC"); + } + if (!firstLine.startsWith(WARC_MAGIC)) { + throw new IOException("Failed to find WARC MAGIC: " + firstLine); + } + // Here we start reading off the inputstream but we're reading the + // stream direct rather than going via WARCRecord#read. The latter will + // keep count of bytes read, digest and fail properly if EOR too soon... + // We don't want digesting while reading Headers. + // + Header [] h = HttpParser.parseHeaders(in, WARC_HEADER_ENCODING); + for (int i = 0; i < h.length; i++) { + m.put(h[i].getName(), h[i].getValue()); + } + int headerLength = -1; + if (in instanceof RepositionableStream) { + headerLength = + (int)(((RepositionableStream)in).position() - startPosition); + } + final int contentOffset = headerLength; + incrementPosition(contentOffset); + + return new ArchiveRecordHeader() { + private Map headers = m; + private int contentBegin = contentOffset; + + public String getDate() { + return (String)this.headers.get(HEADER_KEY_DATE); + } + + public String getDigest() { + return null; + // TODO: perhaps return block-digest? + // superclass def implies this is calculated ("only after + // read in totality"), not pulled from header, so + // below prior implementation was misleading +// return (String)this.headers.get(HEADER_KEY_CHECKSUM); + } + + public String getReaderIdentifier() { + return (String)this.headers.get(READER_IDENTIFIER_FIELD_KEY); + } + + public Set getHeaderFieldKeys() { + return this.headers.keySet(); + } + + public Map getHeaderFields() { + return this.headers; + } + + public Object getHeaderValue(String key) { + return this.headers.get(key); + } + + public long getLength() { + Object o = this.headers.get(CONTENT_LENGTH); + if (o == null) { + return -1; + } + long contentLength = (o instanceof Long)? + ((Long)o).longValue(): Long.parseLong((String)o); + return contentLength + contentOffset; + } + + public String getMimetype() { + return (String)this.headers.get(CONTENT_TYPE); + } + + public long getOffset() { + Object o = this.headers.get(ABSOLUTE_OFFSET_KEY); + if (o == null) { + return -1; + } + return (o instanceof Long)? + ((Long)o).longValue(): Long.parseLong((String)o); + } + + public String getRecordIdentifier() { + return (String)this.headers.get(RECORD_IDENTIFIER_FIELD_KEY); + } + + public String getUrl() { + return (String)this.headers.get(HEADER_KEY_URI); + } + + public String getVersion() { + return (String)this.headers.get(VERSION_FIELD_KEY); + } + + public int getContentBegin() { + return this.contentBegin; + } + + @Override + public String toString() { + return this.headers.toString(); + } + }; + } + + @Override + protected String getMimetype4Cdx(ArchiveRecordHeader h) { + final String m = super.getMimetype4Cdx(h); + // Mimetypes can have spaces in WARCs. Emitting for CDX, just + // squash them for now. Later, quote them since squashing spaces won't + // work for params that have quoted-string values. + Matcher matcher = WHITESPACE.matcher(m); + return matcher.replaceAll(""); + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/io/warc/WARCWriter.java b/commons/src/main/java/org/archive/io/warc/WARCWriter.java new file mode 100644 index 00000000..6be83896 --- /dev/null +++ b/commons/src/main/java/org/archive/io/warc/WARCWriter.java @@ -0,0 +1,443 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io.warc; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import org.archive.io.UTF8Bytes; +import org.archive.io.WriterPoolMember; +import org.archive.uid.GeneratorFactory; +import org.archive.util.ArchiveUtils; +import org.archive.util.anvl.ANVLRecord; +import org.archive.util.anvl.Element; + + +/** + * Experimental WARC implementation. + * + *

Assumption is that the caller is managing access to this + * ExperimentalWARCWriter ensuring only one thread accessing this WARC instance + * at any one time. + * + *

While being written, WARCs have a '.open' suffix appended. + * + * @author stack + * @version $Revision: 4604 $ $Date: 2006-09-05 22:38:18 -0700 (Tue, 05 Sep 2006) $ + */ +public class WARCWriter extends WriterPoolMember +implements WARCConstants { + + /** + * NEWLINE as bytes. + */ + public static byte [] CRLF_BYTES; + static { + try { + CRLF_BYTES = CRLF.getBytes(DEFAULT_ENCODING); + } catch(Exception e) { + e.printStackTrace(); + } + }; + + /** + * Metadata. + * TODO: Exploit writing warcinfo record. Currently unused. + */ + private final List fileMetadata; + + + /** + * Shutdown Constructor + * Has default access so can make instance to test utility methods. + */ + WARCWriter() { + this(null, null, "", "", true, -1, null); + } + + /** + * Constructor. + * Takes a stream. Use with caution. There is no upperbound check on size. + * Will just keep writing. Only pass Streams that are bounded. + * @param serialNo used to generate unique file name sequences + * @param out Where to write. + * @param f File the out is connected to. + * @param cmprs Compress the content written. + * @param a14DigitDate If null, we'll write current time. + * @throws IOException + */ + public WARCWriter(final AtomicInteger serialNo, + final OutputStream out, final File f, + final boolean cmprs, final String a14DigitDate, + final List warcinfoData) + throws IOException { + super(serialNo, out, f, cmprs, a14DigitDate); + this.fileMetadata = warcinfoData; + } + + /** + * Constructor. + * + * @param dirs Where to drop files. + * @param prefix File prefix to use. + * @param cmprs Compress the records written. + * @param maxSize Maximum size for ARC files written. + * @param suffix File tail to use. If null, unused. + * @param warcinfoData File metadata for warcinfo record. + */ + public WARCWriter(final AtomicInteger serialNo, + final List dirs, final String prefix, + final String suffix, final boolean cmprs, + final long maxSize, final List warcinfoData) { + super(serialNo, dirs, prefix, suffix, cmprs, maxSize, + WARC_FILE_EXTENSION); + this.fileMetadata = warcinfoData; + } + + @Override + protected String createFile(File file) throws IOException { + String filename = super.createFile(file); + writeWarcinfoRecord(filename); + return filename; + } + + protected void baseCharacterCheck(final char c, final String parameter) + throws IOException { + // TODO: Too strict? UNICODE control characters? + if (Character.isISOControl(c) || !Character.isValidCodePoint(c)) { + throw new IOException("Contains illegal character 0x" + + Integer.toHexString(c) + ": " + parameter); + } + } + + protected String checkHeaderValue(final String value) + throws IOException { + for (int i = 0; i < value.length(); i++) { + final char c = value.charAt(i); + baseCharacterCheck(c, value); + if (Character.isWhitespace(c)) { + throw new IOException("Contains disallowed white space 0x" + + Integer.toHexString(c) + ": " + value); + } + } + return value; + } + + protected String checkHeaderLineMimetypeParameter(final String parameter) + throws IOException { + StringBuilder sb = new StringBuilder(parameter.length()); + boolean wasWhitespace = false; + for (int i = 0; i < parameter.length(); i++) { + char c = parameter.charAt(i); + if (Character.isWhitespace(c)) { + // Map all to ' ' and collapse multiples into one. + // TODO: Make sure white space occurs in legal location -- + // before parameter or inside quoted-string. + if (wasWhitespace) { + continue; + } + wasWhitespace = true; + c = ' '; + } else { + wasWhitespace = false; + baseCharacterCheck(c, parameter); + } + sb.append(c); + } + + return sb.toString(); + } + + protected String createRecordHeader(final String type, + final String url, final String create14DigitDate, + final String mimetype, final URI recordId, + final ANVLRecord xtraHeaders, final long contentLength) + throws IOException { + final StringBuilder sb = + new StringBuilder(2048/*A SWAG: TODO: Do analysis.*/); + sb.append(WARC_ID).append(CRLF); + sb.append(HEADER_KEY_TYPE).append(COLON_SPACE).append(type). + append(CRLF); + // Do not write a subject-uri if not one present. + if (url != null && url.length() > 0) { + sb.append(HEADER_KEY_URI).append(COLON_SPACE). + append(checkHeaderValue(url)).append(CRLF); + } + sb.append(HEADER_KEY_DATE).append(COLON_SPACE). + append(create14DigitDate).append(CRLF); + if (xtraHeaders != null) { + for (final Iterator i = xtraHeaders.iterator(); i.hasNext();) { + sb.append(i.next()).append(CRLF); + } + } + + sb.append(HEADER_KEY_ID).append(COLON_SPACE).append('<'). + append(recordId.toString()).append('>').append(CRLF); + if (contentLength > 0) { + sb.append(CONTENT_TYPE).append(COLON_SPACE).append( + checkHeaderLineMimetypeParameter(mimetype)).append(CRLF); + } + sb.append(CONTENT_LENGTH).append(COLON_SPACE). + append(Long.toString(contentLength)).append(CRLF); + + return sb.toString(); + } + + /** + * @deprecated Use {@link #writeRecord(String,String,String,String,URI,ANVLRecord,InputStream,long,boolean)} instead + */ + protected void writeRecord(final String type, final String url, + final String create14DigitDate, final String mimetype, + final URI recordId, ANVLRecord xtraHeaders, + final InputStream contentStream, final long contentLength) + throws IOException { + writeRecord(type, url, create14DigitDate, mimetype, recordId, xtraHeaders, contentStream, contentLength, true); + } + + protected void writeRecord(final String type, final String url, + final String create14DigitDate, final String mimetype, + final URI recordId, ANVLRecord xtraHeaders, + final InputStream contentStream, final long contentLength, + boolean enforceLength) + throws IOException { + if (!TYPES_LIST.contains(type)) { + throw new IllegalArgumentException("Unknown record type: " + type); + } + if (contentLength == 0 && + (xtraHeaders == null || xtraHeaders.size() <= 0)) { + throw new IllegalArgumentException("Cannot write record " + + "of content-length zero and base headers only."); + } + + preWriteRecordTasks(); + try { + final String header = createRecordHeader(type, url, + create14DigitDate, mimetype, recordId, xtraHeaders, + contentLength); + // TODO: Revisit endcoding of header. + write(header.getBytes(WARC_HEADER_ENCODING)); + + if (contentStream != null && contentLength > 0) { + // Write out the header/body separator. + write(CRLF_BYTES); // TODO: should this be written even for zero-length? + copyFrom(contentStream, contentLength, enforceLength); + } + + // Write out the two blank lines at end of all records. + write(CRLF_BYTES); + write(CRLF_BYTES); + } finally { + postWriteRecordTasks(); + } + } + + protected URI generateRecordId(final Map qualifiers) + throws IOException { + URI rid = null; + try { + rid = GeneratorFactory.getFactory(). + getQualifiedRecordID(qualifiers); + } catch (URISyntaxException e) { + // Convert to IOE so can let it out. + throw new IOException(e.getMessage()); + } + return rid; + } + + protected URI generateRecordId(final String key, final String value) + throws IOException { + URI rid = null; + try { + rid = GeneratorFactory.getFactory(). + getQualifiedRecordID(key, value); + } catch (URISyntaxException e) { + // Convert to IOE so can let it out. + throw new IOException(e.getMessage()); + } + return rid; + } + + public URI writeWarcinfoRecord(String filename) + throws IOException { + return writeWarcinfoRecord(filename, null); + } + + public URI writeWarcinfoRecord(String filename, final String description) + throws IOException { + // Strip .open suffix if present. + if (filename.endsWith(WriterPoolMember.OCCUPIED_SUFFIX)) { + filename = filename.substring(0, + filename.length() - WriterPoolMember.OCCUPIED_SUFFIX.length()); + } + ANVLRecord record = new ANVLRecord(2); + record.addLabelValue(HEADER_KEY_FILENAME, filename); + if (description != null && description.length() > 0) { + record.addLabelValue(CONTENT_DESCRIPTION, description); + } + // Add warcinfo body. + byte [] warcinfoBody = null; + if (this.fileMetadata == null) { + // TODO: What to write into a warcinfo? What to associate? + warcinfoBody = "TODO: Unimplemented".getBytes(); + } else { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + for (final Iterator i = this.fileMetadata.iterator(); + i.hasNext();) { + baos.write(i.next().toString().getBytes(UTF8Bytes.UTF8)); + } + warcinfoBody = baos.toByteArray(); + } + URI uri = writeWarcinfoRecord("application/warc-fields", record, + new ByteArrayInputStream(warcinfoBody), warcinfoBody.length); + // TODO: If at start of file, and we're writing compressed, + // write out our distinctive GZIP extensions. + return uri; + } + + /** + * Write a warcinfo to current file. + * TODO: Write crawl metadata or pointers to crawl description. + * @param mimetype Mimetype of the fileMetadata block. + * @param namedFields Named fields. Pass null if none. + * @param fileMetadata Metadata about this WARC as RDF, ANVL, etc. + * @param fileMetadataLength Length of fileMetadata. + * @throws IOException + * @return Generated record-id made with + * data: scheme and + * the current filename. + */ + public URI writeWarcinfoRecord(final String mimetype, + final ANVLRecord namedFields, final InputStream fileMetadata, + final long fileMetadataLength) + throws IOException { + final URI recordid = generateRecordId(TYPE, WARCINFO); + writeWarcinfoRecord(ArchiveUtils.getLog14Date(), mimetype, recordid, + namedFields, fileMetadata, fileMetadataLength); + return recordid; + } + + /** + * Write a warcinfo to current file. + * The warcinfo type uses its recordId as its URL. + * @param recordId URI to use for this warcinfo. + * @param create14DigitDate Record creation date as 14 digit date. + * @param mimetype Mimetype of the fileMetadata. + * @param namedFields Named fields. + * @param fileMetadata Metadata about this WARC as RDF, ANVL, etc. + * @param fileMetadataLength Length of fileMetadata. + * @throws IOException + */ + public void writeWarcinfoRecord(final String create14DigitDate, + final String mimetype, final URI recordId, final ANVLRecord namedFields, + final InputStream fileMetadata, final long fileMetadataLength) + throws IOException { + writeRecord(WARCINFO, null, create14DigitDate, mimetype, + recordId, namedFields, fileMetadata, fileMetadataLength, true); + } + + public void writeRequestRecord(final String url, + final String create14DigitDate, final String mimetype, + final URI recordId, + final ANVLRecord namedFields, final InputStream request, + final long requestLength) + throws IOException { + writeRecord(REQUEST, url, create14DigitDate, + mimetype, recordId, namedFields, request, + requestLength, true); + } + + public void writeResourceRecord(final String url, + final String create14DigitDate, final String mimetype, + final ANVLRecord namedFields, final InputStream response, + final long responseLength) + throws IOException { + writeResourceRecord(url, create14DigitDate, mimetype, getRecordID(), + namedFields, response, responseLength); + } + + public void writeResourceRecord(final String url, + final String create14DigitDate, final String mimetype, + final URI recordId, + final ANVLRecord namedFields, final InputStream response, + final long responseLength) + throws IOException { + writeRecord(RESOURCE, url, create14DigitDate, + mimetype, recordId, namedFields, response, + responseLength, true); + } + + public void writeResponseRecord(final String url, + final String create14DigitDate, final String mimetype, + final URI recordId, + final ANVLRecord namedFields, final InputStream response, + final long responseLength) + throws IOException { + writeRecord(RESPONSE, url, create14DigitDate, + mimetype, recordId, namedFields, response, + responseLength, true); + } + + public void writeRevisitRecord(final String url, + final String create14DigitDate, final String mimetype, + final URI recordId, + final ANVLRecord namedFields, final InputStream response, + final long responseLength) + throws IOException { + writeRecord(REVISIT, url, create14DigitDate, + mimetype, recordId, namedFields, response, + responseLength, false); + } + + public void writeMetadataRecord(final String url, + final String create14DigitDate, final String mimetype, + final URI recordId, + final ANVLRecord namedFields, final InputStream metadata, + final long metadataLength) + throws IOException { + writeRecord(METADATA, url, create14DigitDate, + mimetype, recordId, namedFields, metadata, + metadataLength, true); + } + + /** + * Convenience method for getting Record-Ids. + * @return A record ID. + * @throws IOException + */ + public static URI getRecordID() throws IOException { + URI result; + try { + result = GeneratorFactory.getFactory().getRecordID(); + } catch (URISyntaxException e) { + throw new IOException(e.toString()); + } + return result; + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/io/warc/WARCWriterPool.java b/commons/src/main/java/org/archive/io/warc/WARCWriterPool.java new file mode 100644 index 00000000..6638a26b --- /dev/null +++ b/commons/src/main/java/org/archive/io/warc/WARCWriterPool.java @@ -0,0 +1,76 @@ +/* $Id: WARCWriterPool.java 4566 2006-08-31 16:51:41Z stack-sf $ + * + * Created on August 1st, 2006. + * + * Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.io.warc; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.pool.BasePoolableObjectFactory; +import org.archive.io.WriterPool; +import org.archive.io.WriterPoolMember; +import org.archive.io.WriterPoolSettings; + + +/** + * A pool of WARCWriters. + * @author stack + * @version $Revision: 4566 $ $Date: 2006-08-31 09:51:41 -0700 (Thu, 31 Aug 2006) $ + */ +public class WARCWriterPool extends WriterPool { + /** + * Constructor + * @param settings Settings for this pool. + * @param poolMaximumActive + * @param poolMaximumWait + */ + public WARCWriterPool(final WriterPoolSettings settings, + final int poolMaximumActive, final int poolMaximumWait) { + this(new AtomicInteger(), settings, poolMaximumActive, poolMaximumWait); + } + + /** + * Constructor + * @param serial Used to generate unique filename sequences + * @param settings Settings for this pool. + * @param poolMaximumActive + * @param poolMaximumWait + */ + public WARCWriterPool(final AtomicInteger serial, + final WriterPoolSettings settings, + final int poolMaximumActive, final int poolMaximumWait) { + super(serial, new BasePoolableObjectFactory() { + public Object makeObject() throws Exception { + return new WARCWriter(serial, + settings.getOutputDirs(), + settings.getPrefix(), settings.getSuffix(), + settings.isCompressed(), settings.getMaxSize(), + settings.getMetadata()); + } + + public void destroyObject(Object writer) + throws Exception { + ((WriterPoolMember)writer).close(); + super.destroyObject(writer); + } + }, settings, poolMaximumActive, poolMaximumWait); + } +} diff --git a/commons/src/main/java/org/archive/io/warc/WARCWriterTest.java b/commons/src/main/java/org/archive/io/warc/WARCWriterTest.java new file mode 100644 index 00000000..f3ffd413 --- /dev/null +++ b/commons/src/main/java/org/archive/io/warc/WARCWriterTest.java @@ -0,0 +1,463 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io.warc; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import org.archive.io.ArchiveRecord; +import org.archive.io.ArchiveRecordHeader; +import org.archive.io.UTF8Bytes; +import org.archive.io.WriterPoolMember; +import org.archive.io.warc.WARCConstants; +import org.archive.uid.GeneratorFactory; +import org.archive.util.ArchiveUtils; +import org.archive.util.TmpDirTestCase; +import org.archive.util.anvl.ANVLRecord; + +/** + * Test Writer and Reader. + * @author stack + * @version $Date: 2006-08-29 19:35:48 -0700 (Tue, 29 Aug 2006) $ $Version$ + */ +public class WARCWriterTest +extends TmpDirTestCase implements WARCConstants { + private static final AtomicInteger SERIAL_NO = new AtomicInteger(); + + /** + * Prefix to use for ARC files made by JUNIT. + */ + private static final String PREFIX = "IAH"; + + private static final String SOME_URL = "http://www.archive.org/test/"; + + public void testCheckHeaderLineValue() throws Exception { + WARCWriter writer = new WARCWriter(); + writer.checkHeaderValue("one"); + IOException exception = null; + try { + writer.checkHeaderValue("with space"); + } catch(IOException e) { + exception = e; + } + assertNotNull(exception); + exception = null; + try { + writer.checkHeaderValue("with\0x0000controlcharacter"); + } catch(IOException e) { + exception = e; + } + assertNotNull(exception); + } + + public void testMimetypes() throws IOException { + WARCWriter writer = new WARCWriter(); + writer.checkHeaderLineMimetypeParameter("text/xml"); + writer.checkHeaderLineMimetypeParameter("text/xml+rdf"); + assertEquals(writer.checkHeaderLineMimetypeParameter( + "text/plain; charset=SHIFT-JIS"), "text/plain; charset=SHIFT-JIS"); + assertEquals(writer.checkHeaderLineMimetypeParameter( + "multipart/mixed; \r\n boundary=\"simple boundary\""), + "multipart/mixed; boundary=\"simple boundary\""); + } + + public void testWriteRecord() throws IOException { + File [] files = {getTmpDir()}; + + // Write uncompressed. + WARCWriter writer = + new WARCWriter(SERIAL_NO, Arrays.asList(files), + this.getClass().getName(), "suffix", false, -1, null); + writeFile(writer); + + // Write compressed. + writer = new WARCWriter(SERIAL_NO, Arrays.asList(files), + this.getClass().getName(), "suffix", true, -1, null); + writeFile(writer); + } + + private void writeFile(final WARCWriter writer) + throws IOException { + try { + writeWarcinfoRecord(writer); + writeBasicRecords(writer); + } finally { + writer.close(); + writer.getFile().delete(); + } + } + + private void writeWarcinfoRecord(WARCWriter writer) + throws IOException { + ANVLRecord meta = new ANVLRecord(); + meta.addLabelValue("size", "1G"); + meta.addLabelValue("operator", "igor"); + byte [] bytes = meta.getUTF8Bytes(); + writer.writeWarcinfoRecord(ANVLRecord.MIMETYPE, null, + new ByteArrayInputStream(bytes), bytes.length); + } + + protected void writeBasicRecords(final WARCWriter writer) + throws IOException { + ANVLRecord headerFields = new ANVLRecord(); + headerFields.addLabelValue("x", "y"); + headerFields.addLabelValue("a", "b"); + + URI rid = null; + try { + rid = GeneratorFactory.getFactory(). + getQualifiedRecordID(TYPE, METADATA); + } catch (URISyntaxException e) { + // Convert to IOE so can let it out. + throw new IOException(e.getMessage()); + } + final String content = "Any old content."; + for (int i = 0; i < 10; i++) { + String body = i + ". " + content; + byte [] bodyBytes = body.getBytes(UTF8Bytes.UTF8); + writer.writeRecord(METADATA, "http://www.archive.org/", + ArchiveUtils.get14DigitDate(), "no/type", + rid, headerFields, new ByteArrayInputStream(bodyBytes), + (long)bodyBytes.length, true); + } + } + + /** + * @return Generic HTML Content. + */ + protected static String getContent() { + return getContent(null); + } + + /** + * @return Generic HTML Content with mention of passed indexStr + * in title and body. + */ + protected static String getContent(String indexStr) { + String page = (indexStr != null)? "Page #" + indexStr: "Some Page"; + return "HTTP/1.1 200 OK\r\n" + + "Content-Type: text/html\r\n\r\n" + + "" + page + + "" + + "" + page + + ""; + } + + /** + * Write random HTML Record. + * @param w Where to write. + * @param index An index to put into content. + * @return Length of record written. + * @throws IOException + */ + protected int writeRandomHTTPRecord(WARCWriter w, int index) + throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + String indexStr = Integer.toString(index); + byte[] record = (getContent(indexStr)).getBytes(); + int recordLength = record.length; + baos.write(record); + // Add named fields for ip, checksum, and relate the metadata + // and request to the resource field. + ANVLRecord r = new ANVLRecord(1); + r.addLabelValue(NAMED_FIELD_IP_LABEL, "127.0.0.1"); + w.writeResourceRecord( + "http://www.one.net/id=" + indexStr, + ArchiveUtils.get14DigitDate(), + "text/html; charset=UTF-8", + r, + new ByteArrayInputStream(baos.toByteArray()), + recordLength); + return recordLength; + } + + /** + * Fill a WARC with HTML Records. + * @param baseName WARC basename. + * @param compress Whether to compress or not. + * @param maxSize Maximum WARC size. + * @param recordCount How many records. + * @return The written file. + * @throws IOException + */ + private File writeRecords(String baseName, boolean compress, + int maxSize, int recordCount) + throws IOException { + cleanUpOldFiles(baseName); + File [] files = {getTmpDir()}; + WARCWriter w = new WARCWriter(SERIAL_NO, + Arrays.asList(files), baseName + '-' + PREFIX, "", compress, + maxSize, null); + assertNotNull(w); + for (int i = 0; i < recordCount; i++) { + writeRandomHTTPRecord(w, i); + } + w.close(); + assertTrue("Doesn't exist: " + w.getFile().getAbsolutePath(), + w.getFile().exists()); + return w.getFile(); + } + + /** + * Run validation of passed file. + * @param f File to validate. + * @param recordCount Expected count of records. + * @throws FileNotFoundException + * @throws IOException + */ + private void validate(File f, int recordCount) + throws FileNotFoundException, IOException { + WARCReader reader = WARCReaderFactory.get(f); + assertNotNull(reader); + List headers = null; + if (recordCount == -1) { + headers = reader.validate(); + } else { + headers = reader.validate(recordCount); + } + reader.close(); + + // Now, run through each of the records doing absolute get going from + // the end to start. Reopen the arc so no context between this test + // and the previous. + reader = WARCReaderFactory.get(f); + for (int i = headers.size() - 1; i >= 0; i--) { + ArchiveRecordHeader h = (ArchiveRecordHeader)headers.get(i); + ArchiveRecord r = reader.get(h.getOffset()); + String mimeType = r.getHeader().getMimetype(); + assertTrue("Record is bogus", + mimeType != null && mimeType.length() > 0); + } + reader.close(); + + assertTrue("Metadatas not equal", headers.size() == recordCount); + for (Iterator i = headers.iterator(); i.hasNext();) { + ArchiveRecordHeader r = (ArchiveRecordHeader)i.next(); + assertTrue("Record is empty", r.getLength() > 0); + } + } + + public void testWriteRecords() throws IOException { + final int recordCount = 2; + File f = writeRecords("writeRecord", false, DEFAULT_MAX_WARC_FILE_SIZE, + recordCount); + validate(f, recordCount + 1); // Header record. + } + + public void testRandomAccess() throws IOException { + final int recordCount = 3; + File f = writeRecords("writeRecord", true, DEFAULT_MAX_WARC_FILE_SIZE, + recordCount); + WARCReader reader = WARCReaderFactory.get(f); + // Get to second record. Get its offset for later use. + boolean readFirst = false; + String url = null; + long offset = -1; + long totalRecords = 0; + boolean readSecond = false; + for (final Iterator i = reader.iterator(); i.hasNext(); + totalRecords++) { + WARCRecord ar = (WARCRecord)i.next(); + if (!readFirst) { + readFirst = true; + continue; + } + if (!readSecond) { + url = ar.getHeader().getUrl(); + offset = ar.getHeader().getOffset(); + readSecond = true; + } + } + + reader = WARCReaderFactory.get(f, offset); + ArchiveRecord ar = reader.get(); + assertEquals(ar.getHeader().getUrl(), url); + ar.close(); + + // Get reader again. See how iterator works with offset + reader = WARCReaderFactory.get(f, offset); + int count = 0; + for (final Iterator i = reader.iterator(); i.hasNext(); i.next()) { + count++; + } + reader.close(); + assertEquals(totalRecords - 1, count); + } + + public void testWriteRecordCompressed() throws IOException { + final int recordCount = 2; + File arcFile = writeRecords("writeRecordCompressed", true, + DEFAULT_MAX_WARC_FILE_SIZE, recordCount); + validate(arcFile, recordCount + 1 /*Header record*/); + } + + protected WARCWriter createWARCWriter(String NAME, + boolean compress) { + File [] files = {getTmpDir()}; + return new WARCWriter(SERIAL_NO, + Arrays.asList(files), NAME, "", + compress, DEFAULT_MAX_WARC_FILE_SIZE, null); + } + + protected static ByteArrayOutputStream getBaos(String str) + throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + baos.write(str.getBytes()); + return baos; + } + + protected static void writeRecord(WARCWriter w, String url, + String mimetype, int len, ByteArrayOutputStream baos) + throws IOException { + w.writeResourceRecord(url, + ArchiveUtils.get14DigitDate(), + mimetype, + null, + new ByteArrayInputStream(baos.toByteArray()), + len); + } + + protected int iterateRecords(WARCReader r) + throws IOException { + int count = 0; + for (Iterator i = r.iterator(); i.hasNext();) { + ArchiveRecord ar = i.next(); + ar.close(); + if (count != 0) { + assertTrue("Unexpected URL " + ar.getHeader().getUrl(), + ar.getHeader().getUrl().equals(SOME_URL)); + } + count++; + } + return count; + } + + protected WARCWriter createWithOneRecord(String name, + boolean compressed) + throws IOException { + WARCWriter writer = createWARCWriter(name, compressed); + String content = getContent(); + writeRecord(writer, SOME_URL, "text/html", + content.length(), getBaos(content)); + return writer; + } + + public void testSpaceInURL() { + String eMessage = null; + try { + holeyUrl("testSpaceInURL-" + PREFIX, false, " "); + } catch (IOException e) { + eMessage = e.getMessage(); + } + assertTrue("Didn't get expected exception: " + eMessage, + eMessage.startsWith("Contains disallowed")); + } + + public void testTabInURL() { + String eMessage = null; + try { + holeyUrl("testTabInURL-" + PREFIX, false, "\t"); + } catch (IOException e) { + eMessage = e.getMessage(); + } + assertTrue("Didn't get expected exception: " + eMessage, + eMessage.startsWith("Contains illegal")); + } + + protected void holeyUrl(String name, boolean compress, String urlInsert) + throws IOException { + WARCWriter writer = createWithOneRecord(name, compress); + // Add some bytes on the end to mess up the record. + String content = getContent(); + ByteArrayOutputStream baos = getBaos(content); + writeRecord(writer, SOME_URL + urlInsert + "/index.html", "text/html", + content.length(), baos); + writer.close(); + } + + /** + * Write an arc file for other tests to use. + * @param arcdir Directory to write to. + * @param compress True if file should be compressed. + * @return ARC written. + * @throws IOException + */ + public static File createWARCFile(File arcdir, boolean compress) + throws IOException { + File [] files = {arcdir}; + WARCWriter writer = + new WARCWriter(SERIAL_NO, Arrays.asList(files), + "test", "", compress, DEFAULT_MAX_WARC_FILE_SIZE, null); + String content = getContent(); + writeRecord(writer, SOME_URL, "text/html", content.length(), + getBaos(content)); + writer.close(); + return writer.getFile(); + } + +// public void testSpeed() throws IOException { +// ARCWriter writer = createArcWithOneRecord("speed", true); +// // Add a record with a length that is too long. +// String content = getContent(); +// final int count = 100000; +// logger.info("Starting speed write of " + count + " records."); +// for (int i = 0; i < count; i++) { +// writeRecord(writer, SOME_URL, "text/html", content.length(), +// getBaos(content)); +// } +// writer.close(); +// logger.info("Finished speed write test."); +// } + + public void testArcRecordOffsetReads() throws Exception { + // Get an ARC with one record. + WriterPoolMember w = + createWithOneRecord("testArcRecordInBufferStream", true); + w.close(); + // Get reader on said ARC. + WARCReader r = WARCReaderFactory.get(w.getFile()); + final Iterator i = r.iterator(); + // Skip first ARC meta record. + ArchiveRecord ar = i.next(); + i.hasNext(); + // Now we're at first and only record in ARC. + ar = (WARCRecord) i.next(); + // Now try getting some random set of bytes out of it + // at an odd offset (used to fail because we were + // doing bad math to find where in buffer to read). + final byte[] buffer = new byte[17]; + final int maxRead = 4; + int totalRead = 0; + while (totalRead < maxRead) { + totalRead = totalRead + + ar.read(buffer, 13 + totalRead, maxRead - totalRead); + assertTrue(totalRead > 0); + } + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/io/warc/package.html b/commons/src/main/java/org/archive/io/warc/package.html new file mode 100644 index 00000000..f52aa95b --- /dev/null +++ b/commons/src/main/java/org/archive/io/warc/package.html @@ -0,0 +1,38 @@ + + + +org.archive.io.warc package + + +Experimental WARC Writer and Readers. Code and specification subject to change +with no guarantees of backward compatibility: i.e. newer readers +may not be able to parse WARCs written with older writers. This package +contains prototyping code for revision 0.12 of the WARC specification. +See latest revision +for current state (Version 0.10 code and its documentation has been moved into the +v10 subpackage). + + +

Implementation Notes

+

Tools

+

Initial implementations of Arc2Warc and Warc2Arc +tools can be found in the package above this one, at +{@link org.archive.io.Arc2Warc} and {@link org.archive.io.Warc2Arc} +respectively. Pass --help to learn how to use each tool. +

+ +

TODO

+
    +
  • Is MIME-Version header needed? MIME Parsers seem fine without (python email +lib and java mail).
    • +
  • Should we write out a Content-Transfer-Encoding +header (Currently we do not). Need section in spec. explicit about our +interpretation of MIME and deviations (e.g. content-transfer-encoding should +be assumed binary in case of WARCs, multipart is not disallowed but not +encouraged, etc.)
    • +
  • Minor: Do WARC-Version: 0.12 like MIME-Version: 1.0 rather than +WARC/0.12 for lead in to an ARCRecord?
    • +
+ + + diff --git a/commons/src/main/java/org/archive/net/ClientFTP.java b/commons/src/main/java/org/archive/net/ClientFTP.java new file mode 100644 index 00000000..b9a803d0 --- /dev/null +++ b/commons/src/main/java/org/archive/net/ClientFTP.java @@ -0,0 +1,136 @@ +/* ClientFTP.java + * + * $Id$ + * + * Created on Jun 5, 2003 + * + * Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.net; + + +import java.io.IOException; +import java.net.Socket; + +import org.apache.commons.net.ftp.FTPClient; +import org.apache.commons.net.ftp.FTPReply; + + +/** + * Client for FTP operations. This class is necessary only because the + * {@link #_openDataConnection_(int, String)} method is protected in + * the superclass, and we need to call it directly to handle directory + * listings. (The code that provides directory listings in the + * superclass doesn't scale: It reads the entire directory into + * an in-memory list). + * + *

Additionally, "strict" methods are provided for the other operations + * we use. Maddeningly, the superclass never raises exceptions. If an + * FTP operation fails, then the superclass methods generally return false. + * A developer then needs to check the {@link FTP#getReplyCode()} + * method to see what actually went wrong. The "strict" methods provided + * by this class invoke the superclass method, check if the success flag + * is false, and then raise an {@link FTPException} with the value of + * {@link FTP#getReplyCode()}. + * + * @author pjack + */ +public class ClientFTP extends FTPClient { + + + /** + * Constructs a new ClientFTP. + */ + public ClientFTP() { + } + + + /** + * Connects to the FTP server at the given host and port. + * + * @param host the host of the FTP server to connect to + * @param port the port the FTP server listens on + * @throws IOException if the connection cannot be made due to IO error + * @throws FTPException if the server refuses the connection + */ + public void connectStrict(String host, int port) throws IOException { + this.connect(host, port); + int reply = this.getReplyCode(); + if (!FTPReply.isPositiveCompletion(reply)) { + throw new FTPException(reply); + } + } + + + /** + * Login to the FTP server with the given username and password. + * + * @param user the username to login under + * @param pass the password to use + * @throws IOException if a network error occurs + * @throws FTPException if the login is rejected by the server + * @throws org.apache.commons.net.ftp.FTPConnectionClosedException + * if the FTP server prematurely closes the connection (for + * instance, if the client was idle for too long) + */ + public void loginStrict(String user, String pass) throws IOException { + boolean r = this.login(user, pass); + if (!r) { + throw new FTPException(this.getReplyCode()); + } + } + + + /** + * Tells the FTP server to send binary files. + * + * @throws IOException if a network error occurs + * @throws FTPException if the server rejects the command + * @throws org.apache.commons.net.ftp.FTPConnectionClosedException + * if the FTP server prematurely closes the connection (for + * instance, if the client was idle for too long) + */ + public void setBinary() throws IOException { + boolean r = super.setFileType(BINARY_FILE_TYPE); + if (!r) { + throw new FTPException(getReplyCode()); + } + } + + + /** + * Opens a data connection. + * + * @param command the data command (eg, RETR or LIST) + * @param path the path of the file to retrieve + * @return the socket to read data from + * @throws IOException if a network error occurs + * @throws FTPException if a protocol error occurs + */ + public Socket openDataConnection(int command, String path) + throws IOException { + Socket socket = _openDataConnection_(command, path); + if (socket == null) { + throw new FTPException(this.getReplyCode()); + } + return socket; + } + + +} diff --git a/commons/src/main/java/org/archive/net/DownloadURLConnection.java b/commons/src/main/java/org/archive/net/DownloadURLConnection.java new file mode 100644 index 00000000..a735a521 --- /dev/null +++ b/commons/src/main/java/org/archive/net/DownloadURLConnection.java @@ -0,0 +1,134 @@ +/* $Id$ + * + * Created August 11th, 2006 + * + * Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.net; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.URLConnection; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.archive.util.ProcessUtils; +import org.archive.util.ProcessUtils.ProcessResult; + +/** + * An URL Connection that pre-downloads URL reference before passing back a + * Stream reference. When closed, it removes the local download file. + * @author stack + * @version $Date$, $Revision$ + */ +public abstract class DownloadURLConnection extends URLConnection { + private final String CLASSNAME = DownloadURLConnection.class.getName(); + private final Logger LOGGER = Logger.getLogger(CLASSNAME); + private static final File TMPDIR = + new File(System.getProperty("java.io.tmpdir", "/tmp")); + private File downloadFile = null; + + protected DownloadURLConnection(URL u) { + super(u); + } + + protected String getScript() { + return System.getProperty(this.getClass().getName() + ".path", + "UNDEFINED"); + } + + protected String [] getCommand(final URL thisUrl, + final File downloadFile) { + return new String[] {getScript(), thisUrl.getPath(), + downloadFile.getAbsolutePath()}; + } + + /** + * Do script copy to local file. + * File is available via {@link #getFile()}. + * @throws IOException + */ + public void connect() throws IOException { + if (this.connected) { + return; + } + + this.downloadFile = File.createTempFile(CLASSNAME, null, TMPDIR); + try { + String [] cmd = getCommand(this.url, this.downloadFile); + if (LOGGER.isLoggable(Level.FINE)) { + StringBuffer buffer = new StringBuffer(); + for (int i = 0; i < cmd.length; i++) { + if (i > 0) { + buffer.append(" "); + } + buffer.append(cmd[i]); + } + LOGGER.fine("Command: " + buffer.toString()); + } + ProcessResult pr = ProcessUtils.exec(cmd); + if (pr.getResult() != 0) { + LOGGER.info(cmd + " returned non-null " + pr.getResult()); + } + // Assume download went smoothly. + this.connected = true; + } catch (IOException ioe) { + // Clean up my tmp file. + this.downloadFile.delete(); + this.downloadFile = null; + // Rethrow. + throw ioe; + } + } + + public File getFile() { + return this.downloadFile; + } + + protected void setFile(final File f) { + this.downloadFile = f; + } + + public InputStream getInputStream() throws IOException { + if (!this.connected) { + connect(); + } + + // Return BufferedInputStream so 'delegation' is done for me, so + // I don't have to implement all IS methods and pass to my + // 'delegate' instance. + final DownloadURLConnection connection = this; + return new BufferedInputStream(new FileInputStream(this.downloadFile)) { + private DownloadURLConnection ruc = connection; + + public void close() throws IOException { + super.close(); + if (this.ruc != null && this.ruc.getFile()!= null && + this.ruc.getFile().exists()) { + this.ruc.getFile().delete(); + this.ruc.setFile(null); + } + } + }; + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/net/FTPException.java b/commons/src/main/java/org/archive/net/FTPException.java new file mode 100644 index 00000000..8488caae --- /dev/null +++ b/commons/src/main/java/org/archive/net/FTPException.java @@ -0,0 +1,62 @@ +/* FTPException.java + * + * $Id$ + * + * Created on Jun 5, 2003 + * + * Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.net; + +import java.io.IOException; + +/** + * Indicates that a FTP operation failed due to a protocol violation. + * For instance, if authentication fails. + * + * @author pjack + */ +public class FTPException extends IOException { + private static final long serialVersionUID = 1L; + + /** + * The reply code from the FTP server. + */ + private int code; + + /** + * Constructs a new FTPException. + * + * @param code the error code from the FTP server + */ + public FTPException(int code) { + super("FTP error code: " + code); + this.code = code; + } + + + /** + * Returns the error code from the FTP server. + * + * @return the error code from the FTP server + */ + public int getReplyCode() { + return code; + } +} diff --git a/commons/src/main/java/org/archive/net/LaxURI.java b/commons/src/main/java/org/archive/net/LaxURI.java new file mode 100644 index 00000000..6430c8ce --- /dev/null +++ b/commons/src/main/java/org/archive/net/LaxURI.java @@ -0,0 +1,451 @@ +/* LaxURI +* +* $Id$ +* +* Created on Aug 3, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.net; + +import java.util.Arrays; +import java.util.BitSet; + +import org.apache.commons.httpclient.URI; +import org.apache.commons.httpclient.URIException; +import org.apache.commons.httpclient.util.EncodingUtil; + +/** + * URI subclass which allows partial/inconsistent encoding, matching + * the URIs which will be relayed in requests from popular web + * browsers (esp. Mozilla Firefox and MS IE). + * + * @author gojomo + */ +public class LaxURI extends URI { + + private static final long serialVersionUID = 5273922211722239537L; + + final protected static char[] HTTP_SCHEME = {'h','t','t','p'}; + final protected static char[] HTTPS_SCHEME = {'h','t','t','p','s'}; + + protected static final BitSet lax_rel_segment = new BitSet(256); + // Static initializer for lax_rel_segment + static { + lax_rel_segment.or(rel_segment); + lax_rel_segment.set(':'); // allow ':' + // TODO: add additional allowances as need is demonstrated + } + + protected static final BitSet lax_abs_path = new BitSet(256); + static { + lax_abs_path.or(abs_path); + lax_abs_path.set('|'); // tests indicate Firefox (1.0.6) doesn't escape. + } + + protected static final BitSet lax_query = new BitSet(256); + static { + lax_query.or(query); + lax_query.set('{'); // tests indicate FF doesn't escape { in query + lax_query.set('}'); // tests indicate FF doesn't escape } in query + lax_query.set('|'); // tests indicate FF doesn't escape | in query + lax_query.set('['); // tests indicate FF doesn't escape [ in query + lax_query.set(']'); // tests indicate FF doesn't escape ] in query + lax_query.set('^'); // tests indicate FF doesn't escape ^ in query + } + + // passthrough initializers + public LaxURI(String uri, boolean escaped, String charset) + throws URIException { + super(uri,escaped,charset); + } + public LaxURI(URI base, URI relative) throws URIException { + super(base,relative); + } + public LaxURI(String uri, boolean escaped) throws URIException { + super(uri,escaped); + } + public LaxURI() { + super(); + } + + // overridden to use this class's static decode() + public String getURI() throws URIException { + return (_uri == null) ? null : decode(_uri, getProtocolCharset()); + } + + // overridden to use this class's static decode() + public String getPath() throws URIException { + char[] p = getRawPath(); + return (p == null) ? null : decode(p, getProtocolCharset()); + } + + // overridden to use this class's static decode() + public String getPathQuery() throws URIException { + char[] rawPathQuery = getRawPathQuery(); + return (rawPathQuery == null) ? null : decode(rawPathQuery, + getProtocolCharset()); + } + // overridden to use this class's static decode() + protected static String decode(char[] component, String charset) + throws URIException { + if (component == null) { + throw new IllegalArgumentException( + "Component array of chars may not be null"); + } + return decode(new String(component), charset); + } + + // overridden to use IA's LaxURLCodec, which never throws DecoderException + protected static String decode(String component, String charset) + throws URIException { + if (component == null) { + throw new IllegalArgumentException( + "Component array of chars may not be null"); + } + byte[] rawdata = null; + // try { + rawdata = LaxURLCodec.decodeUrlLoose(EncodingUtil + .getAsciiBytes(component)); + // } catch (DecoderException e) { + // throw new URIException(e.getMessage()); + // } + return EncodingUtil.getString(rawdata, charset); + } + + // overidden to lax() the acceptable-char BitSet passed in + protected boolean validate(char[] component, BitSet generous) { + return super.validate(component, lax(generous)); + } + + // overidden to lax() the acceptable-char BitSet passed in + protected boolean validate(char[] component, int soffset, int eoffset, + BitSet generous) { + return super.validate(component, soffset, eoffset, lax(generous)); + } + + /** + * Given a BitSet -- typically one of the URI superclass's + * predefined static variables -- possibly replace it with + * a more-lax version to better match the character sets + * actually left unencoded in web browser requests + * + * @param generous original BitSet + * @return (possibly more lax) BitSet to use + */ + protected BitSet lax(BitSet generous) { + if (generous == rel_segment) { + // Swap in more lax allowable set + return lax_rel_segment; + } + if (generous == abs_path) { + return lax_abs_path; + } + if (generous == query) { + return lax_query; + } + // otherwise, leave as is + return generous; + } + + /** + * Coalesce the _host and _authority fields where + * possible. + * + * In the web crawl/http domain, most URIs have an + * identical _host and _authority. (There is no port + * or user info.) However, the superclass always + * creates two separate char[] instances. + * + * Notably, the lengths of these char[] fields are + * equal if and only if their values are identical. + * This method makes use of this fact to reduce the + * two instances to one where possible, slimming + * instances. + * + * @see org.apache.commons.httpclient.URI#parseAuthority(java.lang.String, boolean) + */ + protected void parseAuthority(String original, boolean escaped) + throws URIException { + super.parseAuthority(original, escaped); + if (_host != null && _authority != null + && _host.length == _authority.length) { + _host = _authority; + } + } + + + /** + * Coalesce _scheme to existing instances, where appropriate. + * + * In the web-crawl domain, most _schemes are 'http' or 'https', + * but the superclass always creates a new char[] instance. For + * these two cases, we replace the created instance with a + * long-lived instance from a static field, saving 12-14 bytes + * per instance. + * + * @see org.apache.commons.httpclient.URI#setURI() + */ + protected void setURI() { + if (_scheme != null) { + if (_scheme.length == 4 && Arrays.equals(_scheme, HTTP_SCHEME)) { + _scheme = HTTP_SCHEME; + } else if (_scheme.length == 5 + && Arrays.equals(_scheme, HTTP_SCHEME)) { + _scheme = HTTPS_SCHEME; + } + } + super.setURI(); + } + + /** + * IA OVERRIDDEN IN LaxURI TO INCLUDE FIX FOR + * http://issues.apache.org/jira/browse/HTTPCLIENT-588 + * + * In order to avoid any possilbity of conflict with non-ASCII characters, + * Parse a URI reference as a String with the character + * encoding of the local system or the document. + *

+ * The following line is the regular expression for breaking-down a URI + * reference into its components. + * 

+     *   ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
+     *    12            3  4          5       6  7        8 9
+     *

+ * For example, matching the above expression to + * http://jakarta.apache.org/ietf/uri/#Related + * results in the following subexpression matches: + * 

+     *               $1 = http:
+     *  scheme    =  $2 = http
+     *               $3 = //jakarta.apache.org
+     *  authority =  $4 = jakarta.apache.org
+     *  path      =  $5 = /ietf/uri/
+     *               $6 = 
+     *  query     =  $7 = 
+     *               $8 = #Related
+     *  fragment  =  $9 = Related
+     *

+ * + * @param original the original character sequence + * @param escaped true if original is escaped + * @throws URIException If an error occurs. + */ + protected void parseUriReference(String original, boolean escaped) + throws URIException { + + // validate and contruct the URI character sequence + if (original == null) { + throw new URIException("URI-Reference required"); + } + + /* @ + * ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))? + */ + String tmp = original.trim(); + + /* + * The length of the string sequence of characters. + * It may not be equal to the length of the byte array. + */ + int length = tmp.length(); + + /* + * Remove the delimiters like angle brackets around an URI. + */ + if (length > 0) { + char[] firstDelimiter = { tmp.charAt(0) }; + if (validate(firstDelimiter, delims)) { + if (length >= 2) { + char[] lastDelimiter = { tmp.charAt(length - 1) }; + if (validate(lastDelimiter, delims)) { + tmp = tmp.substring(1, length - 1); + length = length - 2; + } + } + } + } + + /* + * The starting index + */ + int from = 0; + + /* + * The test flag whether the URI is started from the path component. + */ + boolean isStartedFromPath = false; + int atColon = tmp.indexOf(':'); + int atSlash = tmp.indexOf('/'); + if ((atColon <= 0 && !tmp.startsWith("//")) + || (atSlash >= 0 && atSlash < atColon)) { + isStartedFromPath = true; + } + + /* + * 

+         *     @@@@@@@@
+         *  ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
+         *

+ */ + int at = indexFirstOf(tmp, isStartedFromPath ? "/?#" : ":/?#", from); + if (at == -1) { + at = 0; + } + + /* + * Parse the scheme. + * 

+         *  scheme    =  $2 = http
+         *              @
+         *  ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
+         *

+ */ + if (at > 0 && at < length && tmp.charAt(at) == ':') { + char[] target = tmp.substring(0, at).toLowerCase().toCharArray(); + if (validate(target, scheme)) { + _scheme = target; + } else { + throw new URIException("incorrect scheme"); + } + from = ++at; + } + + /* + * Parse the authority component. + * 

+         *  authority =  $4 = jakarta.apache.org
+         *                  @@
+         *  ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
+         *

+ */ + // Reset flags + _is_net_path = _is_abs_path = _is_rel_path = _is_hier_part = false; + if (0 <= at && at < length && tmp.charAt(at) == '/') { + // Set flag + _is_hier_part = true; + if (at + 2 < length && tmp.charAt(at + 1) == '/' + && !isStartedFromPath) { + // the temporary index to start the search from + int next = indexFirstOf(tmp, "/?#", at + 2); + if (next == -1) { + next = (tmp.substring(at + 2).length() == 0) ? at + 2 + : tmp.length(); + } + parseAuthority(tmp.substring(at + 2, next), escaped); + from = at = next; + // Set flag + _is_net_path = true; + } + if (from == at) { + // Set flag + _is_abs_path = true; + } + } + + /* + * Parse the path component. + * 

+         *  path      =  $5 = /ietf/uri/
+         *                                @@@@@@
+         *  ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
+         *

+ */ + if (from < length) { + // rel_path = rel_segment [ abs_path ] + int next = indexFirstOf(tmp, "?#", from); + if (next == -1) { + next = tmp.length(); + } + if (!_is_abs_path) { + if (!escaped + && prevalidate(tmp.substring(from, next), disallowed_rel_path) + || escaped + && validate(tmp.substring(from, next).toCharArray(), rel_path)) { + // Set flag + _is_rel_path = true; + } else if (!escaped + && prevalidate(tmp.substring(from, next), disallowed_opaque_part) + || escaped + && validate(tmp.substring(from, next).toCharArray(), opaque_part)) { + // Set flag + _is_opaque_part = true; + } else { + // the path component may be empty + _path = null; + } + } + String s = tmp.substring(from, next); + if (escaped) { + setRawPath(s.toCharArray()); + } else { + setPath(s); + } + at = next; + } + + // set the charset to do escape encoding + String charset = getProtocolCharset(); + + /* + * Parse the query component. + * 

+         *  query     =  $7 = 
+         *                                        @@@@@@@@@
+         *  ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
+         *

+ */ + if (0 <= at && at + 1 < length && tmp.charAt(at) == '?') { + int next = tmp.indexOf('#', at + 1); + if (next == -1) { + next = tmp.length(); + } + if (escaped) { + _query = tmp.substring(at + 1, next).toCharArray(); + if (!validate(_query, query)) { + throw new URIException("Invalid query"); + } + } else { + _query = encode(tmp.substring(at + 1, next), allowed_query, charset); + } + at = next; + } + + /* + * Parse the fragment component. + * 

+         *  fragment  =  $9 = Related
+         *                                                   @@@@@@@@
+         *  ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
+         *

+ */ + if (0 <= at && at + 1 <= length && tmp.charAt(at) == '#') { + if (at + 1 == length) { // empty fragment + _fragment = "".toCharArray(); + } else { + _fragment = (escaped) ? tmp.substring(at + 1).toCharArray() + : encode(tmp.substring(at + 1), allowed_fragment, charset); + } + } + + // set this URI. + setURI(); + } + +} diff --git a/commons/src/main/java/org/archive/net/LaxURLCodec.java b/commons/src/main/java/org/archive/net/LaxURLCodec.java new file mode 100644 index 00000000..0a4595ef --- /dev/null +++ b/commons/src/main/java/org/archive/net/LaxURLCodec.java @@ -0,0 +1,164 @@ +/* IAURLCodec +* +* $Id$ +* +* Created on Jul 21, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.net; + +import java.io.ByteArrayOutputStream; +import java.io.UnsupportedEncodingException; +import java.util.BitSet; + +import org.apache.commons.codec.net.URLCodec; + +/** + * @author gojomo + */ +public class LaxURLCodec extends URLCodec { + public static LaxURLCodec DEFAULT = new LaxURLCodec("UTF-8"); + + // passthrough constructor + public LaxURLCodec(String encoding) { + super(encoding); + } + + /** + * Decodes an array of URL safe 7-bit characters into an array of + * original bytes. Escaped characters are converted back to their + * original representation. + * + * Differs from URLCodec.decodeUrl() in that it throws no + * exceptions; bad or incomplete escape sequences are ignored + * and passed into result undecoded. This matches the behavior + * of browsers, which will use inconsistently-encoded URIs + * in HTTP request-lines. + * + * @param bytes array of URL safe characters + * @return array of original bytes + */ + public static final byte[] decodeUrlLoose(byte[] bytes) + { + if (bytes == null) { + return null; + } + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + for (int i = 0; i < bytes.length; i++) { + int b = bytes[i]; + if (b == '+') { + buffer.write(' '); + continue; + } + if (b == '%') { + if(i+2 -1 && l > -1) { + // good encoding + int c = ((u << 4) + l); + buffer.write((char)c); + i += 2; + continue; + } // else: bad encoding digits, leave '%' in place + } // else: insufficient encoding digits, leave '%' in place + } + buffer.write(b); + } + return buffer.toByteArray(); + } + + /** + * A more expansive set of ASCII URI characters to consider as 'safe' to + * leave unencoded, based on actual browser behavior. + */ + public static BitSet EXPANDED_URI_SAFE = new BitSet(256); + static { + // alpha characters + for (int i = 'a'; i <= 'z'; i++) { + EXPANDED_URI_SAFE.set(i); + } + for (int i = 'A'; i <= 'Z'; i++) { + EXPANDED_URI_SAFE.set(i); + } + // numeric characters + for (int i = '0'; i <= '9'; i++) { + EXPANDED_URI_SAFE.set(i); + } + // special chars + EXPANDED_URI_SAFE.set('-'); + EXPANDED_URI_SAFE.set('~'); + EXPANDED_URI_SAFE.set('_'); + EXPANDED_URI_SAFE.set('.'); + EXPANDED_URI_SAFE.set('*'); + EXPANDED_URI_SAFE.set('/'); + EXPANDED_URI_SAFE.set('='); + EXPANDED_URI_SAFE.set('&'); + EXPANDED_URI_SAFE.set('+'); + EXPANDED_URI_SAFE.set(','); + EXPANDED_URI_SAFE.set(':'); + EXPANDED_URI_SAFE.set(';'); + EXPANDED_URI_SAFE.set('@'); + EXPANDED_URI_SAFE.set('$'); + EXPANDED_URI_SAFE.set('!'); + EXPANDED_URI_SAFE.set(')'); + EXPANDED_URI_SAFE.set('('); + // experiments indicate: Firefox (1.0.6) never escapes '%' + EXPANDED_URI_SAFE.set('%'); + // experiments indicate: Firefox (1.0.6) does not escape '|' or ''' + EXPANDED_URI_SAFE.set('|'); + EXPANDED_URI_SAFE.set('\''); + } + + public static BitSet QUERY_SAFE = new BitSet(256); + static { + QUERY_SAFE.or(EXPANDED_URI_SAFE); + // Tests indicate Firefox (1.0.7-1) doesn't escape curlies in query str. + QUERY_SAFE.set('{'); + QUERY_SAFE.set('}'); + // nor any of these: [ ] ^ ? + QUERY_SAFE.set('['); + QUERY_SAFE.set(']'); + QUERY_SAFE.set('^'); + QUERY_SAFE.set('?'); + } + + /** + * Encodes a string into its URL safe form using the specified + * string charset. Unsafe characters are escaped. + * + * This method is analogous to superclass encode() methods, + * additionally offering the ability to specify a different + * 'safe' character set (such as EXPANDED_URI_SAFE). + * + * @param safe BitSet of characters that don't need to be encoded + * @param pString String to encode + * @param cs Name of character set to use + * @return Encoded version of pString. + * @throws UnsupportedEncodingException + */ + public String encode(BitSet safe, String pString, String cs) + throws UnsupportedEncodingException { + if (pString == null) { + return null; + } + return new String(encodeUrl(safe,pString.getBytes(cs)), "US-ASCII"); + } +} diff --git a/commons/src/main/java/org/archive/net/PublicSuffixes.java b/commons/src/main/java/org/archive/net/PublicSuffixes.java new file mode 100644 index 00000000..8294b323 --- /dev/null +++ b/commons/src/main/java/org/archive/net/PublicSuffixes.java @@ -0,0 +1,302 @@ +/* PublicSuffixes.java + * + * $Id: BloomFilter32bitSplit.java 5197 2007-06-06 01:31:46Z gojomo $ + * + * Created on Jun 13, 2007 + * + * Copyright (C) 2007 Internet Archive + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +package org.archive.net; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.io.IOUtils; +import org.archive.util.TextUtils; + +/** + * Utility class for making use of the information about 'public suffixes' at + * http://publicsuffix.org. + * + * The public suffix list (once known as 'effective TLDs') was motivated by the + * need to decide on which broader domains a subdomain was allowed to set + * cookies. For example, a server at 'www.example.com' can set cookies for + * 'www.example.com' or 'example.com' but not 'com'. 'www.example.co.uk' can set + * cookies for 'www.example.co.uk' or 'example.co.uk' but not 'co.uk' or 'uk'. + * The number of rules for all top-level-domains and 2nd- or 3rd- level domains + * has become quite long; essentially the broadest domain a subdomain may assign + * to is the one that was sold/registered to a specific name registrant. + * + * This concept should be useful in other contexts, too. Grouping URIs (or + * queues of URIs to crawl) together with others sharing the same registered + * suffix may be useful for applying the same rules to all, such as assigning + * them to the same queue or crawler in a multi- machine setup. + * + * @author Gojomo + */ +public class PublicSuffixes { + protected static Pattern topmostAssignedSurtPrefixPattern; + protected static String topmostAssignedSurtPrefixRegex; + + /** + * Utility method for dumping a regex String, based on a published public + * suffix list, which matches any SURT-form hostname up through the broadest + * 'private' (assigned/sold) domain-segment. That is, for any of the + * SURT-form hostnames... + * + * com,example, com,example,www, com,example,california,www + * + * ...the regex will match 'com,example,'. + * + * @param args + * @throws IOException + */ + public static void main(String args[]) throws IOException { + + String regex; + + if (args.length == 0 || "=".equals(args[0])) { + // use bundled list + regex = getTopmostAssignedSurtPrefixRegex(); + } else { + // use specified filename + BufferedReader reader = new BufferedReader(new FileReader(args[0])); + regex = getTopmostAssignedSurtPrefixRegex(reader); + IOUtils.closeQuietly(reader); + } + + boolean needsClose = false; + BufferedWriter writer; + if (args.length >= 2) { + // writer to specified file + writer = new BufferedWriter(new FileWriter(args[1])); + needsClose = true; + } else { + // write to stdout + writer = new BufferedWriter(new OutputStreamWriter(System.out)); + } + writer.append(regex); + writer.flush(); + if (needsClose) { + writer.close(); + } + } + + /** + * Reads a file of the format promulgated by publicsuffix.org, ignoring + * comments and '!' exceptions/notations, converting domain segments to + * SURT-ordering. Leaves glob-style '*' wildcarding in place. Returns sorted + * list of unique SURT-ordered prefixes. + * + * @param reader + * @return + * @throws IOException + */ + public static List readPublishedFileToSurtList(BufferedReader reader) + throws IOException { + String line; + List list = new ArrayList(); + while ((line = reader.readLine()) != null) { + + // discard whitespace, empty lines, comments, exceptions + line = line.trim(); + if (line.length() == 0 || line.startsWith("//")) { + continue; + } + // discard utf8 notation after entry + line = line.split("\\s+")[0]; + line = line.toLowerCase(); + + // SURT-order domain segments + String[] segs = line.split("\\."); + StringBuilder surtregex = new StringBuilder(); + for (int i = segs.length - 1; i >= 0; i--) { + if (segs[i].length() > 0) { + // current list has a stray '?' in a .no domain + String fixed = segs[i].replaceAll("\\?", "_"); + // replace '!' with '+' to indicate lookahead-for-exceptions + // (gets those to sort before '*' at later build-step) + fixed = fixed.replaceAll("!", "+"); + surtregex.append(fixed + ","); + } + } + list.add(surtregex.toString()); + } + + Collections.sort(list); + // uniq + String last = ""; + Iterator iter = list.iterator(); + while (iter.hasNext()) { + String s = iter.next(); + if (s.equals(last)) { + iter.remove(); + continue; + } + last = s; +// System.out.println(s); + } + return list; + } + + /** + * Converts SURT-ordered list of public prefixes into a Java regex which + * matches the public-portion "plus one" segment, giving the domain on which + * cookies can be set or other policy grouping should occur. Also adds to + * regex a fallback matcher that for any new/unknown TLDs assumes the + * second-level domain is assignable. (Eg: 'zzz,example,'). + * + * @param list + * @return + */ + private static String surtPrefixRegexFromSurtList(List list) { + StringBuilder regex = new StringBuilder(); + regex.append("(?ix)^\n"); + TreeSet prefixes = new TreeSet(Collections + .reverseOrder()); + prefixes.addAll(list); + prefixes.add("*,"); // for new/unknown TLDs + buildRegex("", regex, prefixes); + regex.append("\n([\\-\\w]+,)"); + String rstring = regex.toString(); + // convert glob-stars to word-char-runs + rstring = rstring.replaceAll("\\*", "[\\\\-\\\\w]+"); + return rstring; + } + + protected static void buildRegex(String stem, StringBuilder regex, + SortedSet prefixes) { + if (prefixes.isEmpty()) { + return; + } + if (prefixes.size() == 1 && prefixes.first().equals(stem)) { + // avoid unnecessary "(?:)" + return; + } + regex.append("(?:"); + if (stem.length() == 0) { + regex.append("\n "); // linebreak-space before first character + } + Iterator iter = prefixes.iterator(); + char c = 0; + while (iter.hasNext()) { + String s = iter.next(); + if (s.length() > stem.length()) { + char d = s.charAt(stem.length()); + + if (d == '+') { + // convert exception to zero-width-positive-lookahead + regex.append("(?=" + s.substring(stem.length() + 1) + ")"); + } else { + if (d == c) { + continue; + } + c = d; + regex.append(c); + String newStem = s.substring(0, stem.length() + 1); + SortedSet tail = prefixes.tailSet(newStem); + SortedSet range = null; + successor: for (String candidate : tail) { + if (!candidate.equals(newStem)) { + range = prefixes.subSet(s, candidate); + break successor; + } + } + if (range == null) { + range = prefixes.tailSet(s); + } + buildRegex(newStem, regex, range); + } + regex.append('|'); + } else { + // empty suffix; insert dummy to be eaten when loop exits + regex.append('@'); + } + } + // eat the trailing '|' (if no empty '@') or dummy + regex.deleteCharAt(regex.length() - 1); + regex.append(')'); + if (stem.length() == 1) { + regex.append('\n'); // linebreak for TLDs + } + } + + public static synchronized Pattern getTopmostAssignedSurtPrefixPattern() { + if (topmostAssignedSurtPrefixPattern == null) { + topmostAssignedSurtPrefixPattern = Pattern + .compile(getTopmostAssignedSurtPrefixRegex()); + } + return topmostAssignedSurtPrefixPattern; + } + + public static synchronized String getTopmostAssignedSurtPrefixRegex() { + if (topmostAssignedSurtPrefixRegex == null) { + // use bundled list + BufferedReader reader = new BufferedReader(new InputStreamReader( + PublicSuffixes.class.getClassLoader().getResourceAsStream( + "effective_tld_names.dat"))); + topmostAssignedSurtPrefixRegex = getTopmostAssignedSurtPrefixRegex(reader); + IOUtils.closeQuietly(reader); + } + return topmostAssignedSurtPrefixRegex; + } + + public static String getTopmostAssignedSurtPrefixRegex(BufferedReader reader) { + List list; + try { + list = readPublishedFileToSurtList(reader); + } catch (IOException e) { + throw new RuntimeException(e); + } + return surtPrefixRegexFromSurtList(list); + } + + /** + * Truncate SURT to its topmost assigned domain segment; that is, + * the public suffix plus one segment, but as a SURT-ordered prefix. + * + * if the pattern doesn't match, the passed-in SURT is returned. + * + * @param surt SURT to truncate + * @return truncated-to-topmost-assigned SURT prefix + */ + public static String reduceSurtToTopmostAssigned(String surt) { + Matcher matcher = TextUtils.getMatcher( + getTopmostAssignedSurtPrefixRegex(), surt); + if (matcher.find()) { + surt = matcher.group(); + } + TextUtils.recycleMatcher(matcher); + return surt; + } +} diff --git a/commons/src/main/java/org/archive/net/UURI.java b/commons/src/main/java/org/archive/net/UURI.java new file mode 100644 index 00000000..278bbd0a --- /dev/null +++ b/commons/src/main/java/org/archive/net/UURI.java @@ -0,0 +1,447 @@ +/* UURI + * + * $Id$ + * + * Created on Apr 18, 2003 + * + * Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.net; + +import java.io.File; +import java.io.Serializable; +import java.net.URI; +import java.net.URISyntaxException; + +import org.apache.commons.httpclient.URIException; +import org.archive.util.SURT; +import org.archive.util.TextUtils; + + +/** + * Usable URI. + * + * This class wraps {@link org.apache.commons.httpclient.URI} adding caching + * and methods. It cannot be instantiated directly. Go via UURIFactory. + * + *

We used to use {@link java.net.URI} for parsing URIs but ran across + * quirky behaviors and bugs. {@link java.net.URI} is not subclassable -- + * its final -- and its unlikely that java.net.URI will change any time soon + * (See Gordon's considered petition here: + * java.net.URI + * should have loose/tolerant/compatibility option (or allow reuse)). + * + *

This class tries to cache calculated strings such as the extracted host + * and this class as a string rather than have the parent class rerun its + * calculation everytime. + * + * @author gojomo + * @author stack + * + * @see org.apache.commons.httpclient.URI + */ +public class UURI extends LaxURI +implements CharSequence, Serializable { + + private static final long serialVersionUID = -1277570889914647093L; + + //private static Logger LOGGER = + // Logger.getLogger(UURI.class.getName()); + + /** + * Consider URIs too long for IE as illegal. + */ + public final static int MAX_URL_LENGTH = 2083; + + public static final String MASSAGEHOST_PATTERN = "^www\\d*\\."; + + /** + * Cache of the host name. + * + * Super class calculates on every call. Profiling shows us spend 30% of + * total elapsed time in URI class. + */ + private transient String cachedHost = null; + + /** + * Cache of this uuri escaped as a string. + * + * Super class calculates on every call. Profiling shows us spend 30% of + * total elapsed time in URI class. + */ + private transient String cachedEscapedURI = null; + + /** + * Cache of this uuri escaped as a string. + * + * Super class calculates on every call. Profiling shows us spend 30% of + * total elapsed time in URI class. + */ + private transient String cachedString = null; + + /** + * Cached authority minus userinfo. + */ + private transient String cachedAuthorityMinusUserinfo = null; + + /** + * Cache of this uuri in SURT format + */ + private transient String surtForm = null; + + // Technically, underscores are disallowed in the domainlabel + // portion of hostname according to rfc2396 but we'll be more + // loose and allow them. See: [ 1072035 ] [uuri] Underscore in + // host messes up port parsing. + static { + hostname.set('_'); + } + + + /** + * Shutdown access to default constructor. + */ + protected UURI() { + super(); + } + + /** + * @param uri String representation of an absolute URI. + * @param escaped If escaped. + * @param charset Charset to use. + * @throws org.apache.commons.httpclient.URIException + */ + protected UURI(String uri, boolean escaped, String charset) + throws URIException { + super(uri, escaped, charset); + normalize(); + } + + /** + * @param relative String representation of URI. + * @param base Parent UURI to use derelativizing. + * @throws org.apache.commons.httpclient.URIException + */ + protected UURI(UURI base, UURI relative) throws URIException { + super(base, relative); + normalize(); + } + + /** + * @param uri String representation of a URI. + * @param escaped If escaped. + * @throws NullPointerException + * @throws URIException + */ + protected UURI(String uri, boolean escaped) throws URIException, NullPointerException { + super(uri,escaped); + normalize(); + } + + /** + * @param uri URI as string that is resolved relative to this UURI. + * @return UURI that uses this UURI as base. + * @throws URIException + */ + public UURI resolve(String uri) + throws URIException { + return resolve(uri, false, // assume not escaped + this.getProtocolCharset()); + } + + /** + * @param uri URI as string that is resolved relative to this UURI. + * @param e True if escaped. + * @return UURI that uses this UURI as base. + * @throws URIException + */ + public UURI resolve(String uri, boolean e) + throws URIException { + return resolve(uri, e, this.getProtocolCharset()); + } + + /** + * @param uri URI as string that is resolved relative to this UURI. + * @param e True if uri is escaped. + * @param charset Charset to use. + * @return UURI that uses this UURI as base. + * @throws URIException + */ + public UURI resolve(String uri, boolean e, String charset) + throws URIException { + return new UURI(this, new UURI(uri, e, charset)); + } + + /** + * Test an object if this UURI is equal to another. + * + * @param obj an object to compare + * @return true if two URI objects are equal + */ + public boolean equals(Object obj) { + + // normalize and test each components + if (obj == this) { + return true; + } + if (!(obj instanceof UURI)) { + return false; + } + UURI another = (UURI) obj; + // scheme + if (!equals(this._scheme, another._scheme)) { + return false; + } + // is_opaque_part or is_hier_part? and opaque + if (!equals(this._opaque, another._opaque)) { + return false; + } + // is_hier_part + // has_authority + if (!equals(this._authority, another._authority)) { + return false; + } + // path + if (!equals(this._path, another._path)) { + return false; + } + // has_query + if (!equals(this._query, another._query)) { + return false; + } + // UURIs do not have fragments + return true; + } + + /** + * Strips www variants from the host. + * + * Strips www[0-9]*\. from the host. If calling getHostBaseName becomes a + * performance issue we should consider adding the hostBasename member that + * is set on initialization. + * + * @return Host's basename. + * @throws URIException + */ + public String getHostBasename() throws URIException { + // caching eliminated because this is rarely used + // (only benefits legacy DomainScope, which should + // be retired). Saves 4-byte object pointer in UURI + // instances. + return (this.getReferencedHost() == null) + ? null + : TextUtils.replaceFirst(MASSAGEHOST_PATTERN, + this.getReferencedHost(), UURIFactory.EMPTY_STRING); + } + + /** + * Returns an alternate, functional String representation -- in this + * case, a String of the URI represented by this UURI instance. + * + * @return + */ + public synchronized String toCustomString() { + if (this.cachedString == null) { + this.cachedString = super.toString(); + coalesceUriStrings(); + } + return this.cachedString; + } + + /** + * Override to cache result + * + * TODO: eliminate, moving most callers to toCustomString, to avoid + * overloading/diluting toString() + * (see http://webteam.archive.org/confluence/display/Heritrix/Preserve+toString%28%29 ) + * @return String representation of this URI + */ + public String toString() { + return toCustomString(); + } + + public synchronized String getEscapedURI() { + if (this.cachedEscapedURI == null) { + this.cachedEscapedURI = super.getEscapedURI(); + coalesceUriStrings(); + } + return this.cachedEscapedURI; + } + + /** + * The two String fields cachedString and cachedEscapedURI are + * usually identical; if so, coalesce into a single instance. + */ + protected void coalesceUriStrings() { + if (this.cachedString != null && this.cachedEscapedURI != null + && this.cachedString.length() == this.cachedEscapedURI.length()) { + // lengths will only be identical if contents are identical + // (deescaping will always shrink length), so coalesce to + // use only single cached instance + this.cachedString = this.cachedEscapedURI; + } + } + + public synchronized String getHost() throws URIException { + if (this.cachedHost == null) { + // If this._host is null, 3.0 httpclient throws + // illegalargumentexception. Don't go there. + if (this._host != null) { + this.cachedHost = super.getHost(); + coalesceHostAuthorityStrings(); + } + } + return this.cachedHost; + } + + /** + * The two String fields cachedHost and cachedAuthorityMinusUserInfo are + * usually identical; if so, coalesce into a single instance. + */ + protected void coalesceHostAuthorityStrings() { + if (this.cachedAuthorityMinusUserinfo != null + && this.cachedHost != null + && this.cachedHost.length() == + this.cachedAuthorityMinusUserinfo.length()) { + // lengths can only be identical if contents + // are identical; use only one instance + this.cachedAuthorityMinusUserinfo = this.cachedHost; + } + } + + /** + * Return the referenced host in the UURI, if any, also extracting the + * host of a DNS-lookup URI where necessary. + * + * @return the target or topic host of the URI + * @throws URIException + */ + public String getReferencedHost() throws URIException { + String referencedHost = this.getHost(); + if(referencedHost==null && this.getScheme().equals("dns")) { + // extract target domain of DNS lookup + String possibleHost = this.getCurrentHierPath(); + if(possibleHost != null && possibleHost.matches("[-_\\w\\.:]+")) { + referencedHost = possibleHost; + } + } + return referencedHost; + } + + /** + * @return Return the 'SURT' format of this UURI + */ + public String getSurtForm() { + if (surtForm == null) { + surtForm = SURT.fromURI(this.toString()); + } + return surtForm; + } + + /** + * Return the authority minus userinfo (if any). + * + * If no userinfo present, just returns the authority. + * + * @return The authority stripped of any userinfo if present. + * @throws URIException + */ + public String getAuthorityMinusUserinfo() + throws URIException { + if (this.cachedAuthorityMinusUserinfo == null) { + String tmp = getAuthority(); + if (tmp != null && tmp.length() > 0) { + int index = tmp.indexOf('@'); + if (index >= 0 && index < tmp.length()) { + tmp = tmp.substring(index + 1); + } + } + this.cachedAuthorityMinusUserinfo = tmp; + coalesceHostAuthorityStrings(); + } + return this.cachedAuthorityMinusUserinfo; + } + + /* (non-Javadoc) + * @see java.lang.CharSequence#length() + */ + public int length() { + return getEscapedURI().length(); + } + + /* (non-Javadoc) + * @see java.lang.CharSequence#charAt(int) + */ + public char charAt(int index) { + return getEscapedURI().charAt(index); + } + + /* (non-Javadoc) + * @see java.lang.CharSequence#subSequence(int, int) + */ + public CharSequence subSequence(int start, int end) { + return getEscapedURI().subSequence(start,end); + } + + /* (non-Javadoc) + * @see java.lang.Comparable#compareTo(java.lang.Object) + */ + public int compareTo(Object arg0) { + return getEscapedURI().compareTo(arg0.toString()); + } + + + + /** + * Test if passed String has likely URI scheme prefix. + * @param possibleUrl URL string to examine. + * @return True if passed string looks like it could be an URL. + */ + public static boolean hasScheme(String possibleUrl) { + boolean result = false; + for (int i = 0; i < possibleUrl.length(); i++) { + char c = possibleUrl.charAt(i); + if (c == ':') { + if (i != 0) { + result = true; + } + break; + } + if (!scheme.get(c)) { + break; + } + } + return result; + } + + /** + * @param pathOrUri A file path or a URI. + * @return Path parsed from passed pathOrUri. + * @throws URISyntaxException + */ + public static String parseFilename(final String pathOrUri) + throws URISyntaxException { + String path = pathOrUri; + if (UURI.hasScheme(pathOrUri)) { + URI url = new URI(pathOrUri); + path = url.getPath(); + } + return (new File(path)).getName(); + } +} diff --git a/commons/src/main/java/org/archive/net/UURIFactory.java b/commons/src/main/java/org/archive/net/UURIFactory.java new file mode 100644 index 00000000..a426e9e9 --- /dev/null +++ b/commons/src/main/java/org/archive/net/UURIFactory.java @@ -0,0 +1,846 @@ +/* UURIFactory + * + * $Id$ + * + * Created on July 16, 2004 + * + * Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.net; + +import gnu.inet.encoding.IDNA; +import gnu.inet.encoding.IDNAException; +import it.unimi.dsi.mg4j.util.MutableString; + +import java.io.UnsupportedEncodingException; +import java.util.Arrays; +import java.util.BitSet; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.httpclient.URI; +import org.apache.commons.httpclient.URIException; +import org.archive.util.TextUtils; + + +/** + * Factory that returns UURIs. + * + * Does escaping and fixup on URIs massaging in accordance with RFC2396 + * and to match browser practice. For example, it removes any + * '..' if first thing in the path as per IE, converts backslashes to forward + * slashes, and discards any 'fragment'/anchor portion of the URI. This + * class will also fail URIs if they are longer than IE's allowed maximum + * length. + * + *

TODO: Test logging. + * + * @author stack + */ +public class UURIFactory extends URI { + + private static final long serialVersionUID = -6146295130382209042L; + + /** + * Logging instance. + */ + private static Logger logger = + Logger.getLogger(UURIFactory.class.getName()); + + /** + * The single instance of this factory. + */ + private static final UURIFactory factory = new UURIFactory(); + + /** + * RFC 2396-inspired regex. + * + * From the RFC Appendix B: + * 

+     * URI Generic Syntax                August 1998
+     *
+     * B. Parsing a URI Reference with a Regular Expression
+     *
+     * As described in Section 4.3, the generic URI syntax is not sufficient
+     * to disambiguate the components of some forms of URI.  Since the
+     * "greedy algorithm" described in that section is identical to the
+     * disambiguation method used by POSIX regular expressions, it is
+     * natural and commonplace to use a regular expression for parsing the
+     * potential four components and fragment identifier of a URI reference.
+     *
+     * The following line is the regular expression for breaking-down a URI
+     * reference into its components.
+     *
+     * ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
+     * 12            3  4          5       6  7        8 9
+     *
+     * The numbers in the second line above are only to assist readability;
+     * they indicate the reference points for each subexpression (i.e., each
+     * paired parenthesis).  We refer to the value matched for subexpression
+     *  as $.  For example, matching the above expression to
+     *
+     * http://www.ics.uci.edu/pub/ietf/uri/#Related
+     *
+     * results in the following subexpression matches:
+     *
+     * $1 = http:
+     * $2 = http
+     * $3 = //www.ics.uci.edu
+     * $4 = www.ics.uci.edu
+     * $5 = /pub/ietf/uri/
+     * $6 = 
+     * $7 = 
+     * $8 = #Related
+     * $9 = Related
+     *
+     * where  indicates that the component is not present, as is
+     * the case for the query component in the above example.  Therefore, we
+     * can determine the value of the four components and fragment as
+     *
+     * scheme    = $2
+     * authority = $4
+     * path      = $5
+     * query     = $7
+     * fragment  = $9
+     *
+ * + * -- + *

Below differs from the rfc regex in that it has java escaping of + * regex characters and we allow a URI made of a fragment only (Added extra + * group so indexing is off by one after scheme). + */ + final public static Pattern RFC2396REGEX = Pattern.compile( + "^(([^:/?#]+):)?((//([^/?#]*))?([^?#]*)(\\?([^#]*))?)?(#(.*))?"); + // 12 34 5 6 7 8 9 A + // 2 1 54 6 87 3 A9 + // 1: scheme + // 2: scheme: + // 3: //authority/path + // 4: //authority + // 5: authority + // 6: path + // 7: ?query + // 8: query + // 9: #fragment + // A: fragment + + public static final String SLASHDOTDOTSLASH = "^(/\\.\\./)+"; + public static final String SLASH = "/"; + public static final String HTTP = "http"; + public static final String HTTP_PORT = ":80"; + public static final String HTTPS = "https"; + public static final String HTTPS_PORT = ":443"; + public static final String DOT = "."; + public static final String EMPTY_STRING = ""; + public static final String NBSP = "\u00A0"; + public static final String SPACE = " "; + public static final String ESCAPED_SPACE = "%20"; + public static final String TRAILING_ESCAPED_SPACE = "^(.*)(%20)+$"; + public static final String PIPE = "|"; + public static final String PIPE_PATTERN = "\\|"; + public static final String ESCAPED_PIPE = "%7C"; + public static final String CIRCUMFLEX = "^"; + public static final String CIRCUMFLEX_PATTERN = "\\^"; + public static final String ESCAPED_CIRCUMFLEX = "%5E"; + public static final String QUOT = "\""; + public static final String ESCAPED_QUOT = "%22"; + public static final String SQUOT = "'"; + public static final String ESCAPED_SQUOT = "%27"; + public static final String APOSTROPH = "`"; + public static final String ESCAPED_APOSTROPH = "%60"; + public static final String LSQRBRACKET = "["; + public static final String LSQRBRACKET_PATTERN = "\\["; + public static final String ESCAPED_LSQRBRACKET = "%5B"; + public static final String RSQRBRACKET = "]"; + public static final String RSQRBRACKET_PATTERN = "\\]"; + public static final String ESCAPED_RSQRBRACKET = "%5D"; + public static final String LCURBRACKET = "{"; + public static final String LCURBRACKET_PATTERN = "\\{"; + public static final String ESCAPED_LCURBRACKET = "%7B"; + public static final String RCURBRACKET = "}"; + public static final String RCURBRACKET_PATTERN = "\\}"; + public static final String ESCAPED_RCURBRACKET = "%7D"; + public static final String BACKSLASH = "\\"; + public static final String BACKSLASH_PATTERN = "\\\\"; + public static final String ESCAPED_BACKSLASH = "%5C"; + public static final String STRAY_SPACING = "[\n\r\t]+"; + public static final String IMPROPERESC_REPLACE = "%25$1"; + public static final String IMPROPERESC = + "%((?:[^\\p{XDigit}])|(?:.[^\\p{XDigit}])|(?:\\z))"; + public static final String COMMERCIAL_AT = "@"; + public static final char PERCENT_SIGN = '%'; + public static final char COLON = ':'; + + /** + * First percent sign in string followed by two hex chars. + */ + public static final String URI_HEX_ENCODING = + "^[^%]*%[\\p{XDigit}][\\p{XDigit}].*"; + + /** + * Authority port number regex. + */ + final static Pattern PORTREGEX = Pattern.compile("(.*:)([0-9]+)$"); + + /** + * Characters we'll accept in the domain label part of a URI + * authority: ASCII letters-digits-hyphen (LDH) plus underscore, + * with single intervening '.' characters. + * + * (We accept '_' because DNS servers have tolerated for many + * years counter to spec; we also accept dash patterns and ACE + * prefixes that will be rejected by IDN-punycoding attempt.) + */ + final static String ACCEPTABLE_ASCII_DOMAIN = + "^(?:[a-zA-Z0-9_-]++(?:\\.)?)++$"; + + /** + * Pattern that looks for case of three or more slashes after the + * scheme. If found, we replace them with two only as mozilla does. + */ + final static Pattern HTTP_SCHEME_SLASHES = + Pattern.compile("^(https?://)/+(.*)"); + + /** + * Pattern that looks for case of two or more slashes in a path. + */ + final static Pattern MULTIPLE_SLASHES = Pattern.compile("//+"); + + /** + * System property key for list of supported schemes. + */ + private static final String SCHEMES_KEY = ".schemes"; + + /** + * System property key for list of purposefully-ignored schemes. + */ + public static final String IGNORED_SCHEMES_KEY = ".ignored-schemes"; + + private String[] schemes = null; + private String[] ignoredSchemes = null; + + public static final int IGNORED_SCHEME = 9999999; + + /** + * Protected constructor. + */ + private UURIFactory() { + super(); + String s = System.getProperty(this.getClass().getName() + SCHEMES_KEY); + if (s != null && s.length() > 0) { + schemes = s.split("[, ]+"); + Arrays.sort(schemes); + } + String ignored = System.getProperty(this.getClass().getName() + IGNORED_SCHEMES_KEY); + if (ignored != null && ignored.length() > 0) { + ignoredSchemes = ignored.split("[, ]+"); + Arrays.sort(ignoredSchemes); + } + } + + /** + * @param uri URI as string. + * @return An instance of UURI + * @throws URIException + */ + public static UURI getInstance(String uri) throws URIException { + return UURIFactory.factory.create(uri); + } + + /** + * @param uri URI as string. + * @param charset Character encoding of the passed uri string. + * @return An instance of UURI + * @throws URIException + */ + public static UURI getInstance(String uri, String charset) + throws URIException { + return UURIFactory.factory.create(uri, charset); + } + + /** + * @param base Base uri to use resolving passed relative uri. + * @param relative URI as string. + * @return An instance of UURI + * @throws URIException + */ + public static UURI getInstance(UURI base, String relative) + throws URIException { + return UURIFactory.factory.create(base, relative); + } + + /** + * Test of whether passed String has an allowed URI scheme. + * First tests if likely scheme suffix. If so, we then test if its one of + * the supported schemes. + * @param possibleUrl URL string to examine. + * @return True if passed string looks like it could be an URL. + */ + public static boolean hasSupportedScheme(String possibleUrl) { + boolean hasScheme = UURI.hasScheme(possibleUrl); + if (!hasScheme || UURIFactory.factory.schemes == null) { + return hasScheme; + } + String tmpStr = possibleUrl.substring(0, possibleUrl.indexOf(':')); + return Arrays.binarySearch(UURIFactory.factory.schemes, tmpStr) >= 0; + } + + /** + * @param uri URI as string. + * @return Instance of UURI. + * @throws URIException + */ + private UURI create(String uri) throws URIException { + return create(uri, UURI.getDefaultProtocolCharset()); + } + + /** + * @param uri URI as string. + * @param charset Original encoding of the string. + * @return Instance of UURI. + * @throws URIException + */ + private UURI create(String uri, String charset) throws URIException { + UURI uuri = new UURI(fixup(uri, null, charset), true, charset); + if (logger.isLoggable(Level.FINE)) { + logger.fine("URI " + uri + + " PRODUCT " + uuri.toString() + + " CHARSET " + charset); + } + return validityCheck(uuri); + } + + /** + * @param base UURI to use as a base resolving relative. + * @param relative Relative URI. + * @return Instance of UURI. + * @throws URIException + */ + private UURI create(UURI base, String relative) throws URIException { + UURI uuri = new UURI(base, new UURI(fixup(relative, base, base.getProtocolCharset()), + true, base.getProtocolCharset())); + if (logger.isLoggable(Level.FINE)) { + logger.fine(" URI " + relative + + " PRODUCT " + uuri.toString() + + " CHARSET " + base.getProtocolCharset() + + " BASE " + base); + } + return validityCheck(uuri); + } + + /** + * Check the generated UURI. + * + * At the least look at length of uuri string. We were seeing case + * where before escaping, string was < MAX_URL_LENGTH but after was + * >. Letting out a too-big message was causing us troubles later + * down the processing chain. + * @param uuri Created uuri to check. + * @return The passed uuri so can easily inline this check. + * @throws URIException + */ + protected UURI validityCheck(UURI uuri) throws URIException { + if (uuri.getRawURI().length > UURI.MAX_URL_LENGTH) { + throw new URIException("Created (escaped) uuri > " + + UURI.MAX_URL_LENGTH +": "+uuri.toString()); + } + return uuri; + } + + /** + * Do heritrix fix-up on passed uri string. + * + * Does heritrix escaping; usually escaping done to make our behavior align + * with IEs. This method codifies our experience pulling URIs from the + * wilds. Its does all the escaping we want; its output can always be + * assumed to be 'escaped' (though perhaps to a laxer standard than the + * vanilla HttpClient URI class or official specs might suggest). + * + * @param uri URI as string. + * @param base May be null. + * @param e True if the uri is already escaped. + * @return A fixed up URI string. + * @throws URIException + */ + private String fixup(String uri, final URI base, final String charset) + throws URIException { + if (uri == null) { + throw new NullPointerException(); + } else if (uri.length() == 0 && base == null) { + throw new URIException("URI length is zero (and not relative)."); + } + + if (uri.length() > UURI.MAX_URL_LENGTH) { + // We check length here and again later after all convertions. + throw new URIException("URI length > " + UURI.MAX_URL_LENGTH + + ": " + uri); + } + + // Replace nbsp with normal spaces (so that they get stripped if at + // ends, or encoded if in middle) + if (uri.indexOf(NBSP) >= 0) { + uri = TextUtils.replaceAll(NBSP, uri, SPACE); + } + + // Get rid of any trailing spaces or new-lines. + uri = uri.trim(); + + // IE actually converts backslashes to slashes rather than to %5C. + // Since URIs that have backslashes usually work only with IE, we will + // convert backslashes to slashes as well. + // TODO: Maybe we can first convert backslashes by specs and than by IE + // so that we fetch both versions. + if (uri.indexOf(BACKSLASH) >= 0) { + uri = TextUtils.replaceAll(BACKSLASH_PATTERN, uri, SLASH); + } + + // Remove stray TAB/CR/LF + uri = TextUtils.replaceAll(STRAY_SPACING, uri, EMPTY_STRING); + + // Test for the case of more than two slashes after the http(s) scheme. + // Replace with two slashes as mozilla does if found. + // See [ 788219 ] URI Syntax Errors stop page parsing. + Matcher matcher = HTTP_SCHEME_SLASHES.matcher(uri); + if (matcher.matches()) { + uri = matcher.group(1) + matcher.group(2); + } + + // now, minimally escape any whitespace + uri = escapeWhitespace(uri); + + // For further processing, get uri elements. See the RFC2396REGEX + // comment above for explaination of group indices used in the below. + matcher = RFC2396REGEX.matcher(uri); + if (!matcher.matches()) { + throw new URIException("Failed parse of " + uri); + } + String uriScheme = checkUriElementAndLowerCase(matcher.group(2)); + String uriSchemeSpecificPart = checkUriElement(matcher.group(3)); + String uriAuthority = checkUriElement(matcher.group(5)); + String uriPath = checkUriElement(matcher.group(6)); + String uriQuery = checkUriElement(matcher.group(8)); + // UNUSED String uriFragment = checkUriElement(matcher.group(10)); + + // If a scheme, is it a supported scheme? + if (uriScheme != null && uriScheme.length() > 0 && + this.schemes != null) { + if (!(Arrays.binarySearch(schemes,uriScheme)>=0)) { + // unsupported; see if silently ignored + if((Arrays.binarySearch(ignoredSchemes,uriScheme)>=0)) { + throw new URIException( + IGNORED_SCHEME, "Ignored scheme: " + uriScheme); + } else { + throw new URIException("Unsupported scheme: " + uriScheme); + } + } + } + + // Test if relative URI. If so, need a base to resolve against. + if (uriScheme == null || uriScheme.length() <= 0) { + if (base == null) { + throw new URIException("Relative URI but no base: " + uri); + } + } else { + checkHttpSchemeSpecificPartSlashPrefix(base, uriScheme, + uriSchemeSpecificPart); + } + + // fixup authority portion: lowercase/IDN-punycode any domain; + // remove stray trailing spaces + uriAuthority = fixupAuthority(uriAuthority); + + // Do some checks if absolute path. + if (uriSchemeSpecificPart != null && + uriSchemeSpecificPart.startsWith(SLASH)) { + if (uriPath != null) { + // Eliminate '..' if its first thing in the path. IE does this. + uriPath = TextUtils.replaceFirst(SLASHDOTDOTSLASH, uriPath, + SLASH); + } + // Ensure root URLs end with '/': browsers always send "/" + // on the request-line, so we should consider "http://host" + // to be "http://host/". + if (uriPath == null || EMPTY_STRING.equals(uriPath)) { + uriPath = SLASH; + } + } + + if (uriAuthority != null) { + if (uriScheme != null && uriScheme.length() > 0 && + uriScheme.equals(HTTP)) { + uriAuthority = checkPort(uriAuthority); + uriAuthority = stripTail(uriAuthority, HTTP_PORT); + } else if (uriScheme != null && uriScheme.length() > 0 && + uriScheme.equals(HTTPS)) { + uriAuthority = checkPort(uriAuthority); + uriAuthority = stripTail(uriAuthority, HTTPS_PORT); + } + // Strip any prefix dot or tail dots from the authority. + uriAuthority = stripTail(uriAuthority, DOT); + uriAuthority = stripPrefix(uriAuthority, DOT); + } else { + // no authority; may be relative. consider stripping scheme + // to work-around org.apache.commons.httpclient.URI bug + // ( http://issues.apache.org/jira/browse/HTTPCLIENT-587 ) + if (uriScheme != null && base != null + && uriScheme.equals(base.getScheme())) { + // uriScheme redundant and will only confound httpclient.URI + uriScheme = null; + } + } + + // Ensure minimal escaping. Use of 'lax' URI and URLCodec + // means minimal escaping isn't necessarily complete/consistent. + // There is a chance such lax encoding will throw exceptions + // later at inconvenient times. + // + // One reason for these bad escapings -- though not the only -- + // is that the page is using an encoding other than the ASCII or the + // UTF-8 that is our default URI encoding. In this case the parent + // class is burping on the passed URL encoding. If the page encoding + // was passed into this factory, the encoding seems to be parsed + // correctly (See the testEscapedEncoding unit test). + // + // This fixup may cause us to miss content. There is the charset case + // noted above. TODO: Look out for cases where we fail other than for + // the above given reason which will be fixed when we address + // '[ 913687 ] Make extractors interrogate for charset'. + + uriPath = ensureMinimalEscaping(uriPath, charset); + uriQuery = ensureMinimalEscaping(uriQuery, charset, + LaxURLCodec.QUERY_SAFE); + + // Preallocate. The '1's and '2's in below are space for ':', + // '//', etc. URI characters. + MutableString s = new MutableString( + ((uriScheme != null)? uriScheme.length(): 0) + + 1 // ';' + + ((uriAuthority != null)? uriAuthority.length(): 0) + + 2 // '//' + + ((uriPath != null)? uriPath.length(): 0) + + 1 // '?' + + ((uriQuery != null)? uriQuery.length(): 0)); + appendNonNull(s, uriScheme, ":", true); + appendNonNull(s, uriAuthority, "//", false); + appendNonNull(s, uriPath, "", false); + appendNonNull(s, uriQuery, "?", false); + return s.toString(); + } + + /** + * If http(s) scheme, check scheme specific part begins '//'. + * @throws URIException + * @see http://www.faqs.org/rfcs/rfc1738.html Section 3.1. Common Internet + * Scheme Syntax + */ + protected void checkHttpSchemeSpecificPartSlashPrefix(final URI base, + final String scheme, final String schemeSpecificPart) + throws URIException { + if (scheme == null || scheme.length() <= 0) { + return; + } + if (!scheme.equals("http") && !scheme.equals("https")) { + return; + } + if ( schemeSpecificPart == null + || !schemeSpecificPart.startsWith("//")) { + // only acceptable if schemes match + if (base == null || !scheme.equals(base.getScheme())) { + throw new URIException( + "relative URI with scheme only allowed for " + + "scheme matching base"); + } + return; + } + if (schemeSpecificPart.length() <= 2) { + throw new URIException("http scheme specific part is " + + "too short: " + schemeSpecificPart); + } + } + + /** + * Fixup 'authority' portion of URI, by removing any stray + * encoded spaces, lowercasing any domain names, and applying + * IDN-punycoding to Unicode domains. + * + * @param uriAuthority the authority string to fix + * @return fixed version + * @throws URIException + */ + private String fixupAuthority(String uriAuthority) throws URIException { + // Lowercase the host part of the uriAuthority; don't destroy any + // userinfo capitalizations. Make sure no illegal characters in + // domainlabel substring of the uri authority. + if (uriAuthority != null) { + // Get rid of any trailing escaped spaces: + // http://www.archive.org%20. Rare but happens. + // TODO: reevaluate: do IE or firefox do such mid-URI space-removal? + // if not, we shouldn't either. + while(uriAuthority.endsWith(ESCAPED_SPACE)) { + uriAuthority = uriAuthority.substring(0,uriAuthority.length()-3); + } + + // lowercase & IDN-punycode only the domain portion + int atIndex = uriAuthority.indexOf(COMMERCIAL_AT); + int portColonIndex = uriAuthority.indexOf(COLON,(atIndex<0)?0:atIndex); + if(atIndex<0 && portColonIndex<0) { + // most common case: neither userinfo nor port + return fixupDomainlabel(uriAuthority); + } else if (atIndex<0 && portColonIndex>-1) { + // next most common: port but no userinfo + String domain = fixupDomainlabel(uriAuthority.substring(0,portColonIndex)); + String port = uriAuthority.substring(portColonIndex); + return domain + port; + } else if (atIndex>-1 && portColonIndex<0) { + // uncommon: userinfo, no port + String userinfo = uriAuthority.substring(0,atIndex+1); + String domain = fixupDomainlabel(uriAuthority.substring(atIndex+1)); + return userinfo + domain; + } else { + // uncommon: userinfo, port + String userinfo = uriAuthority.substring(0,atIndex+1); + String domain = fixupDomainlabel(uriAuthority.substring(atIndex+1,portColonIndex)); + String port = uriAuthority.substring(portColonIndex); + return userinfo + domain + port; + } + } + return uriAuthority; + } + + /** + * Fixup the domain label part of the authority. + * + * We're more lax than the spec. in that we allow underscores. + * + * @param label Domain label to fix. + * @return Return fixed domain label. + * @throws URIException + */ + private String fixupDomainlabel(String label) + throws URIException { + + // apply IDN-punycoding, as necessary + try { + // TODO: optimize: only apply when necessary, or + // keep cache of recent encodings + label = IDNA.toASCII(label); + } catch (IDNAException e) { + if(TextUtils.matches(ACCEPTABLE_ASCII_DOMAIN,label)) { + // domain name has ACE prefix, leading/trailing dash, or + // underscore -- but is still a name we wish to tolerate; + // simply continue + } else { + // problematic domain: neither ASCII acceptable characters + // nor IDN-punycodable, so throw exception + // TODO: change to HeritrixURIException so distinguishable + // from URIExceptions in library code + URIException ue = new URIException(e+" "+label); + ue.initCause(e); + throw ue; + } + } + label = label.toLowerCase(); + return label; + } + + /** + * Ensure that there all characters needing escaping + * in the passed-in String are escaped. Stray '%' characters + * are *not* escaped, as per browser behavior. + * + * @param u String to escape + * @param charset + * @return string with any necessary escaping applied + */ + private String ensureMinimalEscaping(String u, final String charset) { + return ensureMinimalEscaping(u, charset, LaxURLCodec.EXPANDED_URI_SAFE); + } + + /** + * Ensure that there all characters needing escaping + * in the passed-in String are escaped. Stray '%' characters + * are *not* escaped, as per browser behavior. + * + * @param u String to escape + * @param charset + * @param bitset + * @return string with any necessary escaping applied + */ + private String ensureMinimalEscaping(String u, final String charset, + final BitSet bitset) { + if (u == null) { + return null; + } + for (int i = 0; i < u.length(); i++) { + char c = u.charAt(i); + if (!bitset.get(c)) { + try { + u = LaxURLCodec.DEFAULT.encode(bitset, u, charset); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + break; + } + } + return u; + } + + /** + * Escape any whitespace found. + * + * The parent class takes care of the bulk of escaping. But if any + * instance of escaping is found in the URI, then we ask for parent + * to do NO escaping. Here we escape any whitespace found irrespective + * of whether the uri has already been escaped. We do this for + * case where uri has been judged already-escaped only, its been + * incompletly done and whitespace remains. Spaces, etc., in the URI are + * a real pain. Their presence will break log file and ARC parsing. + * @param uri URI string to check. + * @return uri with spaces escaped if any found. + */ + protected String escapeWhitespace(String uri) { + // Just write a new string anyways. The perl '\s' is not + // as inclusive as the Character.isWhitespace so there are + // whitespace characters we could miss. So, rather than + // write some awkward regex, just go through the string + // a character at a time. Only create buffer first time + // we find a space. + MutableString buffer = null; + for (int i = 0; i < uri.length(); i++) { + char c = uri.charAt(i); + if (Character.isWhitespace(c)) { + if (buffer == null) { + buffer = new MutableString(uri.length() + + 2 /*If space, two extra characters (at least)*/); + buffer.append(uri.substring(0, i)); + } + buffer.append("%"); + String hexStr = Integer.toHexString(c); + if ((hexStr.length() % 2) > 0) { + buffer.append("0"); + } + buffer.append(hexStr); + + } else { + if (buffer != null) { + buffer.append(c); + } + } + } + return (buffer != null)? buffer.toString(): uri; + } + + /** + * Check port on passed http authority. Make sure the size is not larger + * than allowed: See the 'port' definition on this + * page, http://www.kerio.com/manual/wrp/en/418.htm. + * Also, we've seen port numbers of '0080' whose leading zeros confuse + * the parent class. Strip the leading zeros. + * + * @param uriAuthority + * @return Null or an amended port number. + * @throws URIException + */ + private String checkPort(String uriAuthority) + throws URIException { + Matcher m = PORTREGEX.matcher(uriAuthority); + if (m.matches()) { + String no = m.group(2); + if (no != null && no.length() > 0) { + // First check if the port has leading zeros + // as in '0080'. Strip them if it has and + // then reconstitute the uriAuthority. Be careful + // of cases where port is '0' or '000'. + while (no.charAt(0) == '0' && no.length() > 1) { + no = no.substring(1); + } + uriAuthority = m.group(1) + no; + // Now makesure the number is legit. + int portNo = Integer.parseInt(no); + if (portNo <= 0 || portNo > 65535) { + throw new URIException("Port out of bounds: " + + uriAuthority); + } + } + } + return uriAuthority; + } + + /** + * @param b Buffer to append to. + * @param str String to append if not null. + * @param substr Suffix or prefix to use if str is not null. + * @param suffix True if substr is a suffix. + */ + private void appendNonNull(MutableString b, String str, String substr, + boolean suffix) { + if (str != null && str.length() > 0) { + if (!suffix) { + b.append(substr); + } + b.append(str); + if (suffix) { + b.append(substr); + } + } + } + + /** + * @param str String to work on. + * @param prefix Prefix to strip if present. + * @return str w/o prefix. + */ + private String stripPrefix(String str, String prefix) { + return str.startsWith(prefix)? + str.substring(prefix.length(), str.length()): + str; + } + + /** + * @param str String to work on. + * @param tail Tail to strip if present. + * @return str w/o tail. + */ + private static String stripTail(String str, String tail) { + return str.endsWith(tail)? + str.substring(0, str.length() - tail.length()): + str; + } + + /** + * @param element to examine. + * @return Null if passed null or an empty string otherwise + * element. + */ + private String checkUriElement(String element) { + return (element == null || element.length() <= 0)? null: element; + } + + /** + * @param element to examine and lowercase if non-null. + * @return Null if passed null or an empty string otherwise + * element lowercased. + */ + private String checkUriElementAndLowerCase(String element) { + String tmp = checkUriElement(element); + return (tmp != null)? tmp.toLowerCase(): tmp; + } +} diff --git a/commons/src/main/java/org/archive/net/md5/Handler.java b/commons/src/main/java/org/archive/net/md5/Handler.java new file mode 100644 index 00000000..d7e69137 --- /dev/null +++ b/commons/src/main/java/org/archive/net/md5/Handler.java @@ -0,0 +1,91 @@ +/* $Id$ + * + * Created August 11th, 2006 + * + * Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.net.md5; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.URLConnection; +import java.net.URLStreamHandler; + +/** + * A protocol handler for an 'md5' URI scheme. + * Md5 URLs look like this: md5:deadbeefdeadbeefdeadbeefdeadbeef + * When this handler is invoked against an md5 URL, it passes the raw md5 to + * the configured script as an argument. The configured script then does the + * work to bring the item pointed to by the md5 local so we can open a Stream + * on the local copy. Local file is deleted when we finish. Do + * {@link org.archive.net.DownloadURLConnection#getFile()} to get name of + * temporary file. + * + *

You need to define the system property + * -Djava.protocol.handler.pkgs=org.archive.net to add this handler + * to the java.net.URL set. Also define system properties + * -Dorg.archive.net.md5.Md5URLConnection.path=PATH_TO_SCRIPT to + * pass path of script to run as well as + * -Dorg.archive.net.md5.Md5URLConnection.options=OPTIONS for + * any options you'd like to include. The pointed-to PATH_TO_SCRIPT + * will be invoked as follows: PATH_TO_SCRIPT OPTIONS MD5 + * LOCAL_TMP_FILE. The LOCAL_TMP_FILE file is made in + * java.io.tmpdir using java tmp name code. + * @author stack + */ +public class Handler extends URLStreamHandler { + protected URLConnection openConnection(URL u) { + return new Md5URLConnection(u); + } + + /** + * Main dumps rsync file to STDOUT. + * @param args + * @throws IOException + */ + public static void main(String[] args) + throws IOException { + if (args.length != 1) { + System.out.println("Usage: java java " + + "-Djava.protocol.handler.pkgs=org.archive.net " + + "org.archive.net.md5.Handler " + + "md5:deadbeefdeadbeefdeadbeefdeadbeef"); + System.exit(1); + } + System.setProperty("org.archive.net.md5.Md5URLConnection.path", + "/tmp/manifest"); + System.setProperty("java.protocol.handler.pkgs", "org.archive.net"); + URL u = new URL(args[0]); + URLConnection connect = u.openConnection(); + // Write download to stdout. + final int bufferlength = 4096; + byte [] buffer = new byte [bufferlength]; + InputStream is = connect.getInputStream(); + try { + for (int count = is.read(buffer, 0, bufferlength); + (count = is.read(buffer, 0, bufferlength)) != -1;) { + System.out.write(buffer, 0, count); + } + System.out.flush(); + } finally { + is.close(); + } + } +} diff --git a/commons/src/main/java/org/archive/net/md5/Md5URLConnection.java b/commons/src/main/java/org/archive/net/md5/Md5URLConnection.java new file mode 100644 index 00000000..2cff14f4 --- /dev/null +++ b/commons/src/main/java/org/archive/net/md5/Md5URLConnection.java @@ -0,0 +1,38 @@ +/* $Id$ + * + * Created August 11th, 2006 + * + * Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.net.md5; + +import java.net.URL; + +import org.archive.net.DownloadURLConnection; + +/** + * Md5 URL connection. + * @author stack + * @version $Date$, $Revision$ + */ +public class Md5URLConnection extends DownloadURLConnection { + protected Md5URLConnection(URL u) { + super(u); + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/net/rsync/Handler.java b/commons/src/main/java/org/archive/net/rsync/Handler.java new file mode 100644 index 00000000..06b7e1d5 --- /dev/null +++ b/commons/src/main/java/org/archive/net/rsync/Handler.java @@ -0,0 +1,77 @@ +/* RsyncProtocolHandler.java + * + * $Id$ + * + * Created Jul 15, 2005 + * + * Copyright (C) 2005 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.net.rsync; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.URLConnection; +import java.net.URLStreamHandler; + +/** + * A protocol handler that uses native rsync client to do copy. + * You need to define the system property + * -Djava.protocol.handler.pkgs=org.archive.net to add this handler + * to the java.net.URL set. Assumes rsync is in path. Define + * system property + * -Dorg.archive.net.rsync.RsyncUrlConnection.path=PATH_TO_RSYNC to + * pass path to rsync. Downloads to java.io.tmpdir. + * @author stack + */ +public class Handler extends URLStreamHandler { + protected URLConnection openConnection(URL u) { + return new RsyncURLConnection(u); + } + + /** + * Main dumps rsync file to STDOUT. + * @param args + * @throws IOException + */ + public static void main(String[] args) + throws IOException { + if (args.length != 1) { + System.out.println("Usage: java java " + + "-Djava.protocol.handler.pkgs=org.archive.net " + + "org.archive.net.rsync.Handler RSYNC_URL"); + System.exit(1); + } + URL u = new URL(args[0]); + URLConnection connect = u.openConnection(); + // Write download to stdout. + final int bufferlength = 4096; + byte [] buffer = new byte [bufferlength]; + InputStream is = connect.getInputStream(); + try { + for (int count = is.read(buffer, 0, bufferlength); + (count = is.read(buffer, 0, bufferlength)) != -1;) { + System.out.write(buffer, 0, count); + } + System.out.flush(); + } finally { + is.close(); + } + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/net/rsync/RsyncURLConnection.java b/commons/src/main/java/org/archive/net/rsync/RsyncURLConnection.java new file mode 100644 index 00000000..71126f5f --- /dev/null +++ b/commons/src/main/java/org/archive/net/rsync/RsyncURLConnection.java @@ -0,0 +1,57 @@ +/* RsyncURLConnection.java + * + * $Id$ + * + * Created Jul 19, 2005 + * + * Copyright (C) 2005 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.net.rsync; + +import java.io.File; +import java.net.URL; + +import org.archive.net.DownloadURLConnection; + +/** + * Rsync URL connection. + * @author stack + * @version $Date$, $Revision$ + */ +public class RsyncURLConnection extends DownloadURLConnection { + private final String RSYNC_TIMEOUT = + System.getProperty(RsyncURLConnection.class.getName() + ".timeout", + "300"); + + protected RsyncURLConnection(URL u) { + super(u); + } + + protected String getScript() { + return System.getProperty(this.getClass().getName() + ".path", + "rsync"); + } + + @Override + protected String[] getCommand(final URL thisUrl, + final File downloadFile) { + return new String[] {getScript(), "--timeout=" + RSYNC_TIMEOUT, + this.url.getPath(), downloadFile.getAbsolutePath()}; + } +} diff --git a/commons/src/main/java/org/archive/net/s3/Handler.java b/commons/src/main/java/org/archive/net/s3/Handler.java new file mode 100644 index 00000000..c7063925 --- /dev/null +++ b/commons/src/main/java/org/archive/net/s3/Handler.java @@ -0,0 +1,40 @@ +/* $Id$ + * + * Created October 28th, 2006 + * + * Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.net.s3; + +import java.net.URL; +import java.net.URLConnection; +import java.net.URLStreamHandler; + +/** + * Handler for Amazon S3 URLs of the form + * + * s3://id:secret@bucket/key + * + * @author jlee + */ +public class Handler extends URLStreamHandler { + public URLConnection openConnection(URL url) { + return new S3URLConnection(url); + } +} diff --git a/commons/src/main/java/org/archive/net/s3/S3URLConnection.java b/commons/src/main/java/org/archive/net/s3/S3URLConnection.java new file mode 100644 index 00000000..e991766e --- /dev/null +++ b/commons/src/main/java/org/archive/net/s3/S3URLConnection.java @@ -0,0 +1,88 @@ +package org.archive.net.s3; + +import java.io.InputStream; +import java.io.IOException; +import java.net.URL; +import java.net.URLConnection; + +import org.jets3t.service.impl.rest.httpclient.RestS3Service; +import org.jets3t.service.security.AWSCredentials; +import org.jets3t.service.model.S3Bucket; +import org.jets3t.service.model.S3Object; +import org.jets3t.service.S3ServiceException; + +/** + * URLConnection for Amazon S3 objects. + * + * @author jlee + */ +public class S3URLConnection extends URLConnection { + String id = ""; + String secret = ""; + + S3Object object = null; + + /** + * Contruct a new S3URLConnection. + * + * @param a URL of the form s3://id:secret@bucket/key + */ + public S3URLConnection(URL url) { + super(url); + + String[] userInfo = url.getUserInfo().split(":"); + + if (userInfo != null) { + id = userInfo[0]; + secret = userInfo[1]; + } + } + + /** + * Connect to S3 and get the object reference, but don't read any of + * the object data yet. + */ + public void connect() throws IOException { + try { + AWSCredentials credentials = new AWSCredentials(id, secret); + + RestS3Service service = new RestS3Service(credentials); + + S3Bucket bucket = new S3Bucket(url.getHost()); + + object = service.getObject(bucket, url.getPath().substring(1)); + } catch (S3ServiceException s3e) { + s3e.printStackTrace(); + throw new IOException("Error connecting to S3: " + s3e.getStackTrace()); + } + } + + /** + * XXX Not sure what this should be or if it even matters for our use. + * + * @return the made up content type "arc" + */ + public String getContentType() { + return "arc"; + } + + /** + * Get an InputStream for the object, connecting to S3 if connect() + * hasn't been called yet. + * + * @return InputStream for the S3 object + */ + public InputStream getInputStream() throws IOException { + try { + if (! connected) { + connect(); + } + + return object.getDataInputStream(); + } + catch (S3ServiceException s3e) { + s3e.printStackTrace(); + throw new IOException("Error reading from S3: " + s3e.getStackTrace()); + } + } +} diff --git a/commons/src/main/java/org/archive/queue/StoredQueue.java b/commons/src/main/java/org/archive/queue/StoredQueue.java new file mode 100644 index 00000000..69c137f5 --- /dev/null +++ b/commons/src/main/java/org/archive/queue/StoredQueue.java @@ -0,0 +1,182 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.queue; + +import java.io.Serializable; +import java.util.AbstractQueue; +import java.util.Iterator; +import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Logger; + +import org.archive.bdb.BdbModule; + +import com.sleepycat.bind.EntryBinding; +import com.sleepycat.bind.serial.SerialBinding; +import com.sleepycat.bind.serial.StoredClassCatalog; +import com.sleepycat.bind.tuple.TupleBinding; +import com.sleepycat.collections.StoredSortedMap; +import com.sleepycat.je.Database; +import com.sleepycat.je.DatabaseException; + +/** + * Queue backed by a JE Collections StoredSortedMap. + * + * @author gojomo + * + * @param + */ +public class StoredQueue extends AbstractQueue implements Serializable { + private static final long serialVersionUID = 3L; + private static final Logger logger = + Logger.getLogger(StoredQueue.class.getName()); + + transient StoredSortedMap queueMap; // Long -> E + transient Database queueDb; // Database + AtomicLong tailIndex; // next spot for insert + AtomicLong headIndex; // next spot for read + + /** + * Create a StoredQueue backed by the given Database. + * + * The Class of values to be queued may be provided; there is only a + * benefit when a primitive type is specified. A StoredClassCatalog + * must be provided if a primitive type is not supplied. + * + * @param db + * @param clsOrNull + * @param classCatalog + */ + public StoredQueue(Database db, Class clsOrNull, StoredClassCatalog classCatalog) { + tailIndex = new AtomicLong(0); + headIndex = new AtomicLong(0); + hookupDatabase(db, clsOrNull, classCatalog); + } + + /** + * @param db + * @param clsOrNull + * @param classCatalog + */ + public void hookupDatabase(Database db, Class clsOrNull, StoredClassCatalog classCatalog) { + EntryBinding valueBinding = TupleBinding.getPrimitiveBinding(clsOrNull); + if(valueBinding == null) { + valueBinding = new SerialBinding(classCatalog, clsOrNull); + } + queueDb = db; + queueMap = new StoredSortedMap( + db, + TupleBinding.getPrimitiveBinding(Long.class), + valueBinding, + true); + } + + @Override + public Iterator iterator() { + return queueMap.values().iterator(); + } + + @Override + public int size() { + synchronized(tailIndex) { + synchronized(headIndex) { + return (int)(tailIndex.get()-headIndex.get()); + } + } + + } + + public boolean offer(E o) { + synchronized (tailIndex) { + queueMap.put(tailIndex.getAndIncrement(), o); + } + return true; + } + + @SuppressWarnings("unchecked") + public E peek() { + synchronized (headIndex) { + E head = null; + while(head == null && headIndex.get() < tailIndex.get()) { + head = (E) queueMap.get(headIndex.get()); + if(head != null) { + return head; + } + // ERROR; should never be null with headIndex < tailIndex + logger.severe("unexpected empty index of StoredQueue: " + + headIndex.get() + " (tailIndex: " + + tailIndex.get()); + headIndex.incrementAndGet(); + } + return head; + } + } + + @SuppressWarnings("unchecked") + public E poll() { + synchronized (headIndex) { + E head = peek(); + if(head!=null) { + return (E) queueMap.remove(headIndex.getAndIncrement()); + } else { + return null; + } + } + } + + /** + * A suitable DatabaseConfig for the Database backing a StoredQueue. + * (However, it is not necessary to use these config options.) + * + * @return DatabaseConfig suitable for queue + */ + public static BdbModule.BdbConfig databaseConfig() { + BdbModule.BdbConfig dbConfig = new BdbModule.BdbConfig(); + dbConfig.setTransactional(false); + dbConfig.setAllowCreate(true); + return dbConfig; + } + + /** + * Save the state to a stream (that is, serialize it). + * + * @serialData The capacity is emitted (int), followed by all of + * its elements (each an Object) in the proper order, + * followed by a null + * @param s the stream + */ + private void writeObject(java.io.ObjectOutputStream s) + throws java.io.IOException { + try { + queueDb.sync(); + } catch (DatabaseException e) { + throw new RuntimeException(e); + } + s.defaultWriteObject(); + } + + public void close() { + try { + queueDb.sync(); + queueDb.close(); + } catch (DatabaseException e) { + throw new RuntimeException(e); + } + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/queue/StoredQueueTest.java b/commons/src/main/java/org/archive/queue/StoredQueueTest.java new file mode 100644 index 00000000..ea2537e3 --- /dev/null +++ b/commons/src/main/java/org/archive/queue/StoredQueueTest.java @@ -0,0 +1,152 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.queue; + +import java.io.File; +import java.util.NoSuchElementException; +import java.util.concurrent.LinkedBlockingQueue; + +import org.apache.commons.io.FileUtils; +import org.archive.bdb.BdbModule; +import org.archive.util.TmpDirTestCase; +import org.archive.util.bdbje.EnhancedEnvironment; + +import com.sleepycat.je.Database; +import com.sleepycat.je.DatabaseException; +import com.sleepycat.je.EnvironmentConfig; + +public class StoredQueueTest extends TmpDirTestCase { + StoredQueue queue; + EnhancedEnvironment env; + Database db; + File envDir; + + protected void setUp() throws Exception { + super.setUp(); + this.envDir = new File(getTmpDir(),"StoredMapTest"); + this.envDir.mkdirs(); + try { + EnvironmentConfig envConfig = new EnvironmentConfig(); + envConfig.setTransactional(false); + envConfig.setAllowCreate(true); + env = new EnhancedEnvironment(envDir,envConfig); + BdbModule.BdbConfig dbConfig = StoredQueue.databaseConfig(); + db = env.openDatabase(null, "StoredMapTest", dbConfig.toDatabaseConfig()); + } catch (DatabaseException e) { + throw new RuntimeException(e); + } + this.queue = new StoredQueue(db, String.class, env.getClassCatalog()); + } + + protected void tearDown() throws Exception { + db.close(); + env.close(); + FileUtils.deleteDirectory(this.envDir); + super.tearDown(); + } + + public void testAdd() { + assertEquals("not empty at start",0,queue.size()); + fill(queue, 10); + assertEquals("unexpected size at full",10,queue.size()); + } + + /** + * @deprecated Use {@link #fill(Queue,int)} instead + */ + protected void fill(int size) { + fill(queue, size); + } + + protected void fill(java.util.Queue q, int size) { + for(int i = 1; i <= size; i++) { + q.add("item-"+i); + } + } + + protected int drain(java.util.Queue q) { + int count = 0; + while(true) { + try { + q.remove(); + count++; + } catch(NoSuchElementException nse) { + return count; + } + } + } + + public void testClear() { + fill(queue, 10); + queue.clear(); + assertEquals("unexpected size after clear",0,queue.size()); + } + + public void testRemove() { + fill(queue, 10); + assertEquals("unexpected remove value","item-1",queue.remove()); + assertEquals("improper count of removed items",9,drain(queue)); + try { + queue.remove(); + fail("expected NoSuchElementException not received"); + } catch (NoSuchElementException nse) { + // do nothing + } + } + + public void testOrdering() { + fill(queue, 10); + for(int i = 1; i <= 10; i++) { + assertEquals("unexpected remove value","item-"+i,queue.remove()); + } + } + + public void testElement() { + fill(queue, 10); + assertEquals("unexpected element value","item-1",queue.element()); + assertEquals("unexpected element value",queue.peek(),queue.element()); + queue.clear(); + try { + queue.element(); + fail("expected NoSuchElementException not received"); + } catch (NoSuchElementException nse) { + // do nothing + } + } + + public void xestTimingsAgainstLinkedBlockingQueue() { + tryTimings(50000); + tryTimings(500000); + } + + private void tryTimings(int i) { + LinkedBlockingQueue lbq = new LinkedBlockingQueue(); + long start = System.currentTimeMillis(); + fill(lbq,i); + drain(lbq); + long finish = System.currentTimeMillis(); + System.out.println("LBQ - "+i+":"+(finish-start)); + start = System.currentTimeMillis(); + fill(queue,i); + drain(queue); + finish = System.currentTimeMillis(); + System.out.println("SQ - "+i+":"+(finish-start)); + } +} diff --git a/commons/src/main/java/org/archive/spring/BeanFieldsPatternValidator.java b/commons/src/main/java/org/archive/spring/BeanFieldsPatternValidator.java new file mode 100644 index 00000000..84a7aafb --- /dev/null +++ b/commons/src/main/java/org/archive/spring/BeanFieldsPatternValidator.java @@ -0,0 +1,78 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.spring; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.springframework.beans.BeanWrapperImpl; +import org.springframework.validation.Errors; +import org.springframework.validation.Validator; + +public class BeanFieldsPatternValidator implements Validator { + public class PropertyPatternRule { + String propertyName; + Pattern requiredPattern; + String errorMessage; + public PropertyPatternRule(String name, String pat, String msg) { + propertyName = name; + requiredPattern = Pattern.compile(pat); + errorMessage = msg.replace("@@", pat); + } + public void test(BeanWrapperImpl wrapper, Errors errors) { + Matcher m = requiredPattern.matcher( + (CharSequence)wrapper.getPropertyValue(propertyName)); + if(!m.matches()) { + errors.rejectValue(propertyName, null, errorMessage); + } + } + + } + + Class clazz; + List rules; + + public BeanFieldsPatternValidator(Class clazz, String ... fieldsPatterns) { + this.clazz = clazz; + if((fieldsPatterns.length % 3)!=0) { + throw new IllegalArgumentException( + "variable arguments must be multiple of 3"); + } + rules = new ArrayList(); + for(int i = 0; i < fieldsPatterns.length; i=i+3) { + rules.add(new PropertyPatternRule(fieldsPatterns[i],fieldsPatterns[i+1],fieldsPatterns[i+2])); + } + } + + @SuppressWarnings("unchecked") + public boolean supports(Class cls) { + return this.clazz.isAssignableFrom(cls); + } + + public void validate(Object target, Errors errors) { + BeanWrapperImpl w = new BeanWrapperImpl(target); + for(PropertyPatternRule rule : rules) { + rule.test(w,errors); + } + } + +} diff --git a/commons/src/main/java/org/archive/spring/ConfigFile.java b/commons/src/main/java/org/archive/spring/ConfigFile.java new file mode 100644 index 00000000..2662ab47 --- /dev/null +++ b/commons/src/main/java/org/archive/spring/ConfigFile.java @@ -0,0 +1,67 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.spring; + +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.Reader; +import java.io.Writer; + +import org.archive.io.ReadSource; + +/** + * ConfigPath with added implication that it is an individual, + * readable/writable File. + */ +public class ConfigFile extends ConfigPath implements ReadSource, WriteTarget { + private static final long serialVersionUID = 1L; + + public ConfigFile() { + super(); + } + + public ConfigFile(String name, String path) { + super(name, path); + } + + public Reader getReader() { + try { + if(!getFile().exists()) { + getFile().createNewFile(); + } + return new FileReader(getFile()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public Writer getWriter() { + return getWriter(false); + } + + public Writer getWriter(boolean append) { + try { + return new FileWriter(getFile(), append); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/commons/src/main/java/org/archive/spring/ConfigPath.java b/commons/src/main/java/org/archive/spring/ConfigPath.java new file mode 100644 index 00000000..5476b48b --- /dev/null +++ b/commons/src/main/java/org/archive/spring/ConfigPath.java @@ -0,0 +1,113 @@ +/* This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software! + * + * Copyright 2008, Internet Archive Heritrix Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $Header$ + */ +package org.archive.spring; + +import java.io.File; +import java.io.Serializable; + +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import org.springframework.beans.factory.annotation.Required; + +/** + * A filesystem path, as a bean, for the convenience of configuration + * via srping beans.xml or user interfaces to same. + * + * Adds an optional relative-to base path and symbolic handle. + * + * See also ConfigPath + */ +public class ConfigPath implements Serializable { + private static final long serialVersionUID = 1L; + + String name; + String path; + ConfigPath base; + transient File resolved; + + + public ConfigPath() { + super(); + } + + public ConfigPath(String name, String path) { + super(); + this.name = name; + this.path = path; + } + + public ConfigPath getBase() { + return base; + } + + public void setBase(ConfigPath base) { + this.base = base; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getPath() { + return path; + } + + @Required + public void setPath(String path) { + this.path = path; + } + + public File getFile() { + return (base == null) + ? new File(path) + : new File(base.getFile(), path); + } + + public ConfigPath merge(ConfigPath previous) { + if(name==null) { + setName(previous.getName()); + } + if(path==null) { + setPath(previous.getPath()); + } + return this; + } +} diff --git a/commons/src/main/java/org/archive/spring/ConfigPathConfigurer.java b/commons/src/main/java/org/archive/spring/ConfigPathConfigurer.java new file mode 100644 index 00000000..06239256 --- /dev/null +++ b/commons/src/main/java/org/archive/spring/ConfigPathConfigurer.java @@ -0,0 +1,133 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.spring; + +import java.beans.PropertyDescriptor; +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.lang.StringUtils; +import org.archive.io.ReadSource; +import org.springframework.beans.BeanWrapperImpl; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.config.BeanPostProcessor; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.context.support.AbstractApplicationContext; + +/** + * Bean to fixup all configuration-relative ConfigPath instances, and + * maintain an inventory of referenced paths. + * + * @contributor gojomo + */ +public class ConfigPathConfigurer + implements BeanPostProcessor, ApplicationContextAware { + + //// BEANPOSTPROCESSOR IMPLEMENTATION + /** + * Fix all beans with ConfigPath properties that lack a base path + * or a name, to use a job-implied base path and name. + * @see org.springframework.beans.factory.config.BeanPostProcessor#postProcessAfterInitialization(java.lang.Object, java.lang.String) + */ + public Object postProcessAfterInitialization(Object bean, String beanName) + throws BeansException { + fixupPaths(bean, beanName); + return bean; + + } + + protected Object fixupPaths(Object bean, String beanName) { + BeanWrapperImpl wrapper = new BeanWrapperImpl(bean); + for(PropertyDescriptor d : wrapper.getPropertyDescriptors()) { + if(ConfigPath.class.isAssignableFrom(d.getPropertyType()) + || ReadSource.class.isAssignableFrom(d.getPropertyType())) { + Object value = wrapper.getPropertyValue(d.getName()); + if(ConfigPath.class.isInstance(value)) { + ConfigPath cp = (ConfigPath) value; + if(cp==null) { + continue; + } + if(cp.getBase()==null) { + cp.setBase(path); + } + if(StringUtils.isEmpty(cp.getName())) { + cp.setName(beanName+"."+d.getName()); + } + remember(cp); + } + } + } + return bean; + } + + //// BEAN PROPERTIES + + /** 'home' directory for all other paths to be resolved + * relative to; defaults to directory of primary XML config file */ + ConfigPath path; + public ConfigPath getPath() { + return path; + } + public void setPath(ConfigPath p) { + path = p; + } + + //// APPLICATIONCONTEXTAWARE IMPLEMENTATION + + AbstractApplicationContext appCtx; + /** + * Remember ApplicationContext, and if possible primary + * configuration file's home directory. + * @see org.springframework.context.ApplicationContextAware#setApplicationContext(org.springframework.context.ApplicationContext) + */ + public void setApplicationContext(ApplicationContext appCtx) throws BeansException { + this.appCtx = (AbstractApplicationContext)appCtx; + String basePath; + if(appCtx instanceof PathSharingContext) { + String primaryConfigurationPath = ((PathSharingContext)appCtx).getPrimaryConfigurationPath(); + if(primaryConfigurationPath.startsWith("file:")) { + // strip URI-scheme if present (as is usual) + primaryConfigurationPath = primaryConfigurationPath.substring(5); + } + File configFile = new File(primaryConfigurationPath); + basePath = configFile.getParent(); + } else { + basePath = "."; + } + path = new ConfigPath("job base",basePath); + } + + // REMEMBERED PATHS + Map paths = new HashMap(); + protected void remember(ConfigPath cp) { + paths.put(cp.getName(), cp); + } + public Map getPaths() { + return paths; + } + + // noop + public Object postProcessBeforeInitialization(Object bean, String beanName) + throws BeansException { + return bean; + } +} diff --git a/commons/src/main/java/org/archive/spring/ConfigPathEditor.java b/commons/src/main/java/org/archive/spring/ConfigPathEditor.java new file mode 100644 index 00000000..cd096a49 --- /dev/null +++ b/commons/src/main/java/org/archive/spring/ConfigPathEditor.java @@ -0,0 +1,72 @@ +package org.archive.spring; + +import java.awt.Component; +import java.awt.Graphics; +import java.awt.Rectangle; +import java.beans.PropertyChangeListener; +import java.beans.PropertyEditor; + + +public class ConfigPathEditor implements PropertyEditor { + Object value; + + public void addPropertyChangeListener(PropertyChangeListener listener) { + // TODO Auto-generated method stub + + } + + public String getAsText() { + // TODO Auto-generated method stub + return null; + } + + public Component getCustomEditor() { + // TODO Auto-generated method stub + return null; + } + + public String getJavaInitializationString() { + // TODO Auto-generated method stub + return null; + } + + public String[] getTags() { + // TODO Auto-generated method stub + return null; + } + + public Object getValue() { + ConfigPath c = new ConfigPath(null,value.toString()); + //c.put(value); + return c; + } + + public boolean isPaintable() { + // TODO Auto-generated method stub + return false; + } + + public void paintValue(Graphics gfx, Rectangle box) { + // TODO Auto-generated method stub + + } + + public void removePropertyChangeListener(PropertyChangeListener listener) { + // TODO Auto-generated method stub + + } + + public void setAsText(String text) throws IllegalArgumentException { + setValue(text); + } + + public void setValue(Object value) { + this.value = value; + } + + public boolean supportsCustomEditor() { + // TODO Auto-generated method stub + return false; + } + +} diff --git a/commons/src/main/java/org/archive/spring/ConfigString.java b/commons/src/main/java/org/archive/spring/ConfigString.java new file mode 100644 index 00000000..956a7a53 --- /dev/null +++ b/commons/src/main/java/org/archive/spring/ConfigString.java @@ -0,0 +1,19 @@ +package org.archive.spring; + +import java.io.Reader; +import java.io.StringReader; + +import org.archive.io.ReadSource; + +public class ConfigString implements ReadSource { + String value; + public String getValue() { + return value; + } + public void setValue(String value) { + this.value = value; + } + public Reader getReader() { + return new StringReader(value); + } +} diff --git a/commons/src/main/java/org/archive/spring/HasKeyedProperties.java b/commons/src/main/java/org/archive/spring/HasKeyedProperties.java new file mode 100644 index 00000000..f674328a --- /dev/null +++ b/commons/src/main/java/org/archive/spring/HasKeyedProperties.java @@ -0,0 +1,11 @@ +package org.archive.spring; + +/** + * Interface indicating an object has an internal map of properties, + * and thus at least partially amenable to sheet-based contextual + * overriding of properties. + * + */ +public interface HasKeyedProperties { + public KeyedProperties getKeyedProperties(); +} diff --git a/commons/src/main/java/org/archive/spring/HasValidator.java b/commons/src/main/java/org/archive/spring/HasValidator.java new file mode 100644 index 00000000..8a24146a --- /dev/null +++ b/commons/src/main/java/org/archive/spring/HasValidator.java @@ -0,0 +1,7 @@ +package org.archive.spring; + +import org.springframework.validation.Validator; + +public interface HasValidator { + Validator getValidator(); +} diff --git a/commons/src/main/java/org/archive/spring/KeyedProperties.java b/commons/src/main/java/org/archive/spring/KeyedProperties.java new file mode 100644 index 00000000..a8c38439 --- /dev/null +++ b/commons/src/main/java/org/archive/spring/KeyedProperties.java @@ -0,0 +1,144 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package org.archive.spring; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * Map for storing overridable properties. + * + * An object wanting to allow its properties to be overridden + * contextually will store those properties in this map. Its + * accessors (like getProp() and setProp()) will only pass-through + * to the 'prop' entry in this map.) + * + */ +public class KeyedProperties extends HashMap { + private static final long serialVersionUID = 3403222335436162778L; + /** the alternate global property-paths leading to this map + * TODO: consider if deterministic ordered list is important */ + HashSet externalPaths = new HashSet(); + + /** + * Add a path by which the outside world can reach this map + * @param path String path + */ + public void addExternalPath(String path) { + externalPaths.add(path); + } + + /** + * Get the given value, checking override maps if appropriate. + * + * @param key + * @return discovered override, or local value + */ + public Object get(String key) { + for(OverlayContext ocontext: threadOverrides.get()) { + for(String name: ocontext.getOverlayNames()) { + Map m = ocontext.getOverlayMap(name); + for(String ok : getOverrideKeys(key)) { + Object val = m.get(ok); + if(val!=null) { + return val; + } + } + } + } + return super.get(key); + } + + /** + * Compose the complete keys (externalPath + local key name) to use + * for checking for contextual overrides. + * + * @param key local key to compose + * @return List of full keys to check + */ + protected List getOverrideKeys(String key) { + ArrayList keys = new ArrayList(externalPaths.size()); + for(String path : externalPaths) { + keys.add(path+"."+key); + } + return keys; + } + + + // + // CLASS SERVICES + // + + /** + * ThreadLocal (contextual) collection of pushed override maps + */ + static ThreadLocal> threadOverrides = + new ThreadLocal>() { + protected LinkedList initialValue() { + return new LinkedList(); + } + }; + /** + * Add an override map to the stack + * @param m Map to add + */ + static public void pushOverrideContext(OverlayContext ocontext) { + threadOverrides.get().addFirst(ocontext); + } + + /** + * Remove last-added override map from the stack + * @return Map removed + */ + static public OverlayContext popOverridesContext() { + // TODO maybe check that pop is as expected + return threadOverrides.get().removeFirst(); + } + + static public void clearAllOverrideContexts() { + threadOverrides.get().clear(); + } + + static public void loadOverridesFrom(OverlayContext ocontext) { + assert ocontext.haveOverlayNamesBeenSet(); + pushOverrideContext(ocontext); + } + + static public boolean clearOverridesFrom(OverlayContext ocontext) { + return threadOverrides.get().remove(ocontext); + } + + static public void withOverridesDo(OverlayContext ocontext, Runnable todo) { + try { + loadOverridesFrom(ocontext); + todo.run(); + } finally { + clearOverridesFrom(ocontext); + } + } + + public static boolean overridesActiveFrom(OverlayContext ocontext) { + return threadOverrides.get().contains(ocontext); + } +} diff --git a/commons/src/main/java/org/archive/spring/OverlayContext.java b/commons/src/main/java/org/archive/spring/OverlayContext.java new file mode 100644 index 00000000..ebccd0ab --- /dev/null +++ b/commons/src/main/java/org/archive/spring/OverlayContext.java @@ -0,0 +1,38 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package org.archive.spring; + +import java.util.LinkedList; +import java.util.Map; + +/** + * Interface for objects that can contribute 'overlays' to replace the + * usual values in configured objects. + * @contributor gojomo + */ +public interface OverlayContext { + /** test if this context has actually been configured with overlays + * (even if in fact no overlays were added) */ + public boolean haveOverlayNamesBeenSet(); + /** return a list of the names of overlay maps to consider */ + LinkedList getOverlayNames(); + /** get the map corresponding to the overlay name */ + Map getOverlayMap(String name); +} diff --git a/commons/src/main/java/org/archive/spring/OverlayMapsSource.java b/commons/src/main/java/org/archive/spring/OverlayMapsSource.java new file mode 100644 index 00000000..9aef51c3 --- /dev/null +++ b/commons/src/main/java/org/archive/spring/OverlayMapsSource.java @@ -0,0 +1,31 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.spring; + +import java.util.Map; + +/** + * Interface for a source of overlay maps by name. + * + * @contributor gojomo + */ +public interface OverlayMapsSource { + public Map getOverlayMap(String name); +} diff --git a/commons/src/main/java/org/archive/spring/PathSharingContext.java b/commons/src/main/java/org/archive/spring/PathSharingContext.java new file mode 100644 index 00000000..3c4a927b --- /dev/null +++ b/commons/src/main/java/org/archive/spring/PathSharingContext.java @@ -0,0 +1,220 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.spring; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.Map; + +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; +import org.springframework.context.ApplicationContext; +import org.springframework.context.Lifecycle; +import org.springframework.context.event.ContextClosedEvent; +import org.springframework.context.event.ContextStartedEvent; +import org.springframework.context.event.ContextStoppedEvent; +import org.springframework.context.support.FileSystemXmlApplicationContext; +import org.springframework.validation.BeanPropertyBindingResult; +import org.springframework.validation.Errors; +import org.springframework.validation.Validator; + +/** + * Spring ApplicationContext extended for Heritrix use. + * + * Notable extensions: + * + * Remembers its primary XML configuration file, and can report its + * filesystem path. + * + * Propagates lifecycle events (start, stop) without triggering + * loops in the case of circular dependencies. + * + * Reports a summary of Errors collected from self-Validating Beans. + * + * @contributor gojomo + */ +public class PathSharingContext extends FileSystemXmlApplicationContext { + + public PathSharingContext(String configLocation) throws BeansException { + super(configLocation); + // TODO Auto-generated constructor stub + } + + public PathSharingContext(String[] configLocations, ApplicationContext parent) throws BeansException { + super(configLocations, parent); + // TODO Auto-generated constructor stub + } + + public PathSharingContext(String[] configLocations, boolean refresh, ApplicationContext parent) throws BeansException { + super(configLocations, refresh, parent); + // TODO Auto-generated constructor stub + } + + public PathSharingContext(String[] configLocations, boolean refresh) throws BeansException { + super(configLocations, refresh); + // TODO Auto-generated constructor stub + } + + public PathSharingContext(String[] configLocations) throws BeansException { + super(configLocations); + // TODO Auto-generated constructor stub + } + + public String getPrimaryConfigurationPath() { + // TODO Auto-generated method stub + return getConfigLocations()[0]; + } + + @SuppressWarnings("unchecked") + @Override + public void start() { + Map lifecycleBeans = getLifecycleBeans(); + for (Iterator it = new HashSet(lifecycleBeans.keySet()).iterator(); it.hasNext();) { + String beanName = (String) it.next(); + doStart(lifecycleBeans, beanName); + } + publishEvent(new ContextStartedEvent(this)); + } + + @SuppressWarnings("unchecked") + protected void doStart(Map lifecycleBeans, String beanName) { + Lifecycle bean = (Lifecycle) lifecycleBeans.remove(beanName); + if (bean != null) { + String[] dependenciesForBean = getBeanFactory().getDependenciesForBean(beanName); + for (int i = 0; i < dependenciesForBean.length; i++) { + doStart(lifecycleBeans, dependenciesForBean[i]); + } + if (!bean.isRunning()) { + bean.start(); + } + //lifecycleBeans.remove(beanName); + } + } + + @SuppressWarnings("unchecked") + public void stop() { + Map lifecycleBeans = getLifecycleBeans(); + for (Iterator it = new HashSet(lifecycleBeans.keySet()).iterator(); it.hasNext();) { + String beanName = (String) it.next(); + doStop(lifecycleBeans, beanName); + } + publishEvent(new ContextStoppedEvent(this)); + } + + @SuppressWarnings("unchecked") + protected void doStop(Map lifecycleBeans, String beanName) { + Lifecycle bean = (Lifecycle) lifecycleBeans.remove(beanName); + if (bean != null) { + String[] dependentBeans = getBeanFactory().getDependentBeans(beanName); + for (int i = 0; i < dependentBeans.length; i++) { + doStop(lifecycleBeans, dependentBeans[i]); + } + if (bean.isRunning()) { + bean.stop(); + } + //lifecycleBeans.remove(beanName); + } + } + + @SuppressWarnings("unchecked") + protected Map getLifecycleBeans() { + ConfigurableListableBeanFactory beanFactory = getBeanFactory(); + String[] beanNames = beanFactory.getBeanNamesForType(Lifecycle.class, false, false); + Map beans = new HashMap(beanNames.length); + for (int i = 0; i < beanNames.length; i++) { + Object bean = beanFactory.getSingleton(beanNames[i]); + if (bean != null) { + beans.put(beanNames[i], bean); + } + } + return beans; + } + + // override to avoid lifecycle loops via AbstractApplicationContext's + // problematic doStop() + @SuppressWarnings("unchecked") + protected void doClose() { + if (isActive()) { + if (logger.isInfoEnabled()) { + logger.info("Closing " + this); + } + try { + // Publish shutdown event. + publishEvent(new ContextClosedEvent(this)); + } + catch (Throwable ex) { + logger.error("Exception thrown from ApplicationListener handling ContextClosedEvent", ex); + } + // Stop all Lifecycle beans, to avoid delays during individual destruction. + Map lifecycleBeans = getLifecycleBeans(); + for (Iterator it = new LinkedHashSet(lifecycleBeans.keySet()).iterator(); it.hasNext();) { + String beanName = (String) it.next(); + doStop(lifecycleBeans, beanName); + } + // Destroy all cached singletons in the context's BeanFactory. + destroyBeans(); + // Close the state of this context itself. + closeBeanFactory(); + onClose(); + // because these are private to superclass... +// synchronized (this.activeMonitor) { +// this.active = false; +// } + // ... fake it with this eqivalent method... + cancelRefresh(null); + } + } + + + // + // Cascading self-validation + // + + HashMap allErrors; // bean name -> Errors + @SuppressWarnings("unchecked") + public void validate() { + allErrors = new HashMap(); + + for(Object entry : getBeansOfType(HasValidator.class).entrySet()) { + String name = (String) ((Map.Entry)entry).getKey(); + HasValidator hv = (HasValidator) ((Map.Entry)entry).getValue(); + Validator v = hv.getValidator(); + Errors errors = new BeanPropertyBindingResult(hv,name); + v.validate(hv, errors); + if(errors.hasErrors()) { + allErrors.put(name,errors); + } + } + System.err.println("===errors==="); + for(String name : allErrors.keySet()) { + for(Object obj : allErrors.get(name).getAllErrors()) { + System.err.println(name+": "+obj); + } + } + System.err.println("============"); + } + + public HashMap getAllErrors() { + return allErrors; + } + +} diff --git a/commons/src/main/java/org/archive/spring/Sheet.java b/commons/src/main/java/org/archive/spring/Sheet.java new file mode 100644 index 00000000..9b3815ea --- /dev/null +++ b/commons/src/main/java/org/archive/spring/Sheet.java @@ -0,0 +1,132 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.spring; +import java.beans.PropertyChangeEvent; +import java.util.Map; + +import org.springframework.beans.BeanWrapperImpl; +import org.springframework.beans.BeansException; +import org.springframework.beans.TypeMismatchException; +import org.springframework.beans.factory.BeanFactory; +import org.springframework.beans.factory.BeanFactoryAware; +import org.springframework.beans.factory.BeanNameAware; +import org.springframework.beans.factory.annotation.Required; + + +/** + * Collection of overrides: alternative values for object properties + * that should apply in some contexts. The target is specified as an + * arbitrarily-long property-path, a string describing how to access + * the property starting from a beanName in a BeanFactory. + * + * Subclasses implement specific indicators of when an override applies. + */ +public abstract class Sheet implements BeanFactoryAware, BeanNameAware { + private static final long serialVersionUID = 9129011082185864377L; + + String beanName; + BeanFactory beanFactory; + /** map of full property-paths (from BeanFactory to individual + * property) and their changed value when this Sheet of overrides + * is in effect + */ + Map map; + + public void setBeanFactory(BeanFactory beanFactory) throws BeansException { + this.beanFactory = beanFactory; + } + public void setBeanName(String name) { + this.beanName = name; + } + public String getBeanName() { + return beanName; + } + + /** + * Return map of full property-path (starting with a target bean-name) + * to the alternate value for that targeted property + * @return Map + */ + public Map getMap() { + return map; + } + @Required + public void setMap(Map m) { + this.map = m; + } + + /** + * Ensure any properties targetted by this Sheet know to + * check the right property paths for overrides at lookup time, + * and that the override values are compatible types for their + * destination properties. + * + * Should be done as soon as all possible targets are + * constructed (ApplicationListener ContextRefreshedEvent) + * + * TODO: consider if an 'un-priming' also needs to occur to + * prevent confusing side-effects. + * TODO: consider if priming should move to another class + */ + public void prime() { + for (String fullpath : map.keySet()) { + int lastDot = fullpath.lastIndexOf("."); + String beanPath = fullpath.substring(0,lastDot); + String terminalProp = fullpath.substring(lastDot+1); + Object value = map.get(fullpath); + int i = beanPath.indexOf("."); + Object bean; + HasKeyedProperties hkp; + if (i < 0) { + bean = beanFactory.getBean(beanPath); + } else { + String beanName = beanPath.substring(0,i); + String propPath = beanPath.substring(i+1); + BeanWrapperImpl wrapper = new BeanWrapperImpl(beanFactory.getBean(beanName)); + bean = wrapper.getPropertyValue(propPath); + } + try { + hkp = (HasKeyedProperties) bean; + } catch (ClassCastException cce) { + // targetted bean has no overridable properties + throw new TypeMismatchException(bean,HasKeyedProperties.class,cce); + } + // install knowledge of this path + hkp.getKeyedProperties().addExternalPath(beanPath); + // verify type-compatibility + BeanWrapperImpl wrapper = new BeanWrapperImpl(hkp); + Class requiredType = wrapper.getPropertyType(terminalProp); + try { + // convert for destination type + map.put(fullpath, wrapper.convertForProperty(value,terminalProp)); + } catch(TypeMismatchException tme) { + TypeMismatchException tme2 = + new TypeMismatchException( + new PropertyChangeEvent( + hkp, + fullpath, + wrapper.getPropertyValue(terminalProp), + value), requiredType); + tme2.initCause(tme); + throw tme2; + } + } + } +} diff --git a/commons/src/main/java/org/archive/spring/WriteTarget.java b/commons/src/main/java/org/archive/spring/WriteTarget.java new file mode 100644 index 00000000..aa802961 --- /dev/null +++ b/commons/src/main/java/org/archive/spring/WriteTarget.java @@ -0,0 +1,8 @@ +package org.archive.spring; + +import java.io.Writer; + +public interface WriteTarget { + Writer getWriter(); + Writer getWriter(boolean append); +} diff --git a/commons/src/main/java/org/archive/state/ModuleTestBase.java b/commons/src/main/java/org/archive/state/ModuleTestBase.java new file mode 100644 index 00000000..e32d3d11 --- /dev/null +++ b/commons/src/main/java/org/archive/state/ModuleTestBase.java @@ -0,0 +1,209 @@ +/* Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * cd .java + * Created on October 5, 2006 + * + * $Header$ + */ +package org.archive.state; + + +import java.io.File; +import java.io.Serializable; +import java.util.Arrays; + +import junit.framework.TestCase; + +import org.apache.commons.lang.SerializationUtils; + + +/** + * Base class for unit testing Module implementations. + * + * @author pjack + */ +public abstract class ModuleTestBase extends TestCase { + + + /** + * Magical constructor that attempts to auto-create static key field + * descriptions for your module class. + * + *

If {@link #getSourceCodeDir} and {@link #getResourceDir} both return + * non-null values, then the constructor will look in the resources + * directory for an English resource file for the class. If it finds + * one, nothing magical happens. + * + *

Otherwise, the source code for the module being tested is loaded, + * and parsed to extract the JavaDoc descriptions for the static key + * fields. The results are stored in the appropriate English locale file + * in the resource directory. + * + *

Note the parsing is naive; at minimum, you should load the resulting + * locale file and remove any HTML markup. + */ + public ModuleTestBase() { + getSourceCodeDir(); + getResourceDir(); + } + + + /** + * Returns the location of the source code directory for your project. + * This defaults to "src/main/java", which is the standard for projects + * built with maven2. If you use a different source code directory, + * you should override this method. + * + *

If you want to disable automatic key description generation, + * return null from this method. + * + * @return the source code directory for the project + */ + protected File getSourceCodeDir() { + return getProjectDir("src/main/java"); + } + + + /** + * Returns the location of the Java resources directory for your project. + * This defaults to "src/resources/java", which is the standard for projects + * built with maven2. If you use a different source code directory -- + * for instance, if your resources directory is the same as your source + * code directory -- you should override this method. + * + *

If you want to disable automatic key description generation, + * return null from this method. + * + * @return the source code directory for the project + */ + protected File getResourceDir() { + return getProjectDir("src/main/resources"); + } + + + /** + * Returns a project directory for a Heritrix subproject. This is here + * so that the src and resources directories can be found whether the + * unit test is run using maven2 or using Eclipse. The two build systems + * use different working directories. + * + * @param path the path the path to find + * @return the found path + */ + private File getProjectDir(String path) { + File r = new File(path); + if (r.exists()) { + return r; + } + String cname = getClass().getName(); + if (cname.startsWith("org.archive.processors")) { + return new File("modules/" + path); + } + if (cname.startsWith("org.archive.deciderules")) { + return new File("modules/" + path); + } + if (cname.startsWith("org.archive.crawler")) { + return new File("engine/" + path); + } + return null; + } + + /** + * Returns the class of the module to test. Deduces from + * test class name if possible. + * + * @return the class of the module to test + */ + protected Class getModuleClass() { + String myClassName = this.getClass().getCanonicalName(); + if(!myClassName.endsWith("Test")) { + throw new UnsupportedOperationException( + "Cannot get module class of "+myClassName); + } + String moduleClassName = myClassName.substring(0,myClassName.length()-4); + try { + return Class.forName(moduleClassName); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + + /** + * Return an example instance of the module. This is used by + * testSerialization to ensure the module can be serialized. + * + * @return an example instance of the module + * @throws Exception if the module cannot be constructed for any reason + */ + protected Object makeModule() throws Exception { + return getModuleClass().newInstance(); + } + + /** + * Tests that the module can be serialized. The value returned by + * {@link #makeModule} is serialized to a byte array, and then + * deserialized, and then serialized to a second byte array. The results + * are passed to {@link #verifySerialization}, which will simply compare + * the two byte arrays for equality. (That won't always work; see + * that method for details). + * + *

If nothing else, this test is useful for catching NotSerializable + * exceptions for your module or classes it depends on. + * + * @throws Exception if the module cannot be serialized + */ + public void testSerialization() throws Exception { + Object first = makeModule(); + byte[] firstBytes = SerializationUtils.serialize((Serializable)first); + + Object second = SerializationUtils.deserialize(firstBytes); + byte[] secondBytes = SerializationUtils.serialize((Serializable)second); + + Object third = SerializationUtils.deserialize(secondBytes); + byte[] thirdBytes = SerializationUtils.serialize((Serializable)third); + + // HashMap serialization reverses order of items in linked buckets + // each roundtrip -- so don't check one roundtrip, check two +// verifySerialization(first, firstBytes, second, secondBytes); + verifySerialization(first, firstBytes, third, thirdBytes); + } + + + /** + * Verifies that serialization was successful. + * + *

By default, this method simply compares the first and second byte + * arrays for equality. That may not work if you use custom serialization + * -- for instance, if you're serializing a timestamp. If that's the case + * you should override this method to compare the given objects, or to + * simply do nothing. (If this method does nothing, then the + * {@link #testSerialization} test is still useful for catching + * NotSerializable problems). + * + * @param first the first object that was serialized + * @param firstBytes the byte array the first object was serialized to + * @param second the second object that was serialized + * @param secondBytes the byte array the second object was serialized to + * @throws Exception if anyt problem occurs + */ + protected void verifySerialization(Object first, byte[] firstBytes, + Object second, byte[] secondBytes) throws Exception { + assertTrue(Arrays.equals(firstBytes, secondBytes)); + } +} diff --git a/commons/src/main/java/org/archive/surt/SURTTokenizer.java b/commons/src/main/java/org/archive/surt/SURTTokenizer.java new file mode 100644 index 00000000..7f8f71b9 --- /dev/null +++ b/commons/src/main/java/org/archive/surt/SURTTokenizer.java @@ -0,0 +1,177 @@ +/* SURTTokenizer + * + * $Id$ + * + * Created on 3:21:49 PM May 11, 2006. + * + * Copyright (C) 2006 Internet Archive. + * + * This file is part of wayback. + * + * wayback is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * wayback is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with wayback; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.surt; + +import org.apache.commons.httpclient.URIException; +import org.archive.net.UURI; +import org.archive.net.UURIFactory; +import org.archive.util.ArchiveUtils; +import org.archive.util.SURT; + +/** + * provides iterative Url reduction for prefix matching to find ever coarser + * grained URL-specific configuration. Assumes that a prefix binary search is + * being attempted for each returned value. First value is the entire SURT + * url String, with TAB appended. Second removes CGI ARGs. Then each subsequent + * path segment ('/' separated) is removed. Then the login:password, if present + * is removed. Then the port, if not :80 or omitted on the initial URL. Then + * each subsequent authority segment(. separated) is removed. + * + * the nextSearch() method will return null, finally, when no broader searches + * can be attempted on the URL. + * + * @author brad + * @version $Date$, $Revision$ + */ +public class SURTTokenizer { + + private final static String EXACT_SUFFIX = "\t"; + private String remainder; + private boolean triedExact; + private boolean triedFull; + private boolean choppedArgs; + private boolean choppedPath; + private boolean choppedLogin; + private boolean choppedPort; + + /** + * constructor + * + * @param url String URL + * @throws URIException + */ + public SURTTokenizer(final String url) throws URIException { + remainder = getKey(url,false); + } + /** + * update internal state and return the next smaller search string + * for the url + * + * @return string to lookup for prefix match for relevant information. + */ + public String nextSearch() { + if(!triedExact) { + triedExact = true; + //remainder = remainder.substring(0,remainder.length()-1); + return remainder + EXACT_SUFFIX; + } + if(!triedFull) { + triedFull = true; + return remainder; + } + if(!choppedArgs) { + choppedArgs = true; + int argStart = remainder.indexOf('?'); + if(argStart != -1) { + remainder = remainder.substring(0,argStart); + return remainder; + } + } + if(!choppedPath) { + int lastSlash = remainder.lastIndexOf('/'); + if(lastSlash != -1) { + remainder = remainder.substring(0,lastSlash); + if(remainder.endsWith(")")) { + remainder = remainder.substring(0,remainder.length()-1); + } + return remainder; + } + choppedPath = true; + } + if(!choppedLogin) { + choppedLogin = true; + int lastAt = remainder.lastIndexOf('@'); + if(lastAt != -1) { + remainder = remainder.substring(0,lastAt); + if(remainder.endsWith(",")) { + remainder = remainder.substring(0,remainder.length()-1); + } + return remainder; + } + } + if(!choppedPort) { + choppedPort = true; + int lastColon = remainder.lastIndexOf(':'); + if(lastColon != -1) { + remainder = remainder.substring(0,lastColon); + if(remainder.endsWith(",")) { + remainder = remainder.substring(0,remainder.length()-1); + } + return remainder; + } + } + // now just remove ','s + int lastComma = remainder.lastIndexOf(','); + if(lastComma == -1) { + return null; + } + remainder = remainder.substring(0,lastComma); + return remainder; + } + + /** + * @param url + * @return String SURT which will match exactly argument url + * @throws URIException + */ + public static String exactKey(String url) throws URIException { + return getKey(url,false); + } + + /** + * @param url + * @return String SURT which will match urls prefixed with the argument url + * @throws URIException + */ + public static String prefixKey(String url) throws URIException { + return getKey(url,true); + } + + private static String getKey(String url, boolean prefix) + throws URIException { + + String key = ArchiveUtils.addImpliedHttpIfNecessary(url); + UURI uuri = UURIFactory.getInstance(key); + key = uuri.getScheme() + "://" + uuri.getAuthority() + + uuri.getEscapedPathQuery(); + + key = SURT.fromURI(key); + + int hashPos = key.indexOf('#'); + if(hashPos != -1) { + key = key.substring(0,hashPos); + } + + if(key.startsWith("http://")) { + key = key.substring(7); + } + if(prefix) { + if(key.endsWith(")/")) { + key = key.substring(0,key.length()-2); + } + } + return key; + } +} diff --git a/commons/src/main/java/org/archive/uid/Generator.java b/commons/src/main/java/org/archive/uid/Generator.java new file mode 100644 index 00000000..6c6fc744 --- /dev/null +++ b/commons/src/main/java/org/archive/uid/Generator.java @@ -0,0 +1,83 @@ +/* $Id$ + * + * Created on July 27th, 2006 + * + * Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.uid; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Map; + +/** + * A record-id generator. + * {@link GeneratorFactory} assumes implementations have a no-arg Constructor. + * @see GeneratorFactory + * @author stack + * @version $Revision$ $Date$ + */ +public interface Generator { + /** + * @return A URI that can serve as a record-id. + * @throws URISyntaxException + */ + public URI getRecordID() throws URISyntaxException; + + /** + * @param qualifiers Qualifiers to add. + * @return A URI qualified with passed qualifiers that can + * serve as a record-id, or, a new, unique record-id without qualifiers + * (if qualifiers not easily implemented using passed URI scheme). + * @throws URISyntaxException + */ + public URI getQualifiedRecordID(final Map qualifiers) + throws URISyntaxException; + + /** + * @param key Name of qualifier + * @param value Value of qualifier + * @return A URI qualified with passed qualifiers that can + * serve as a record-id, or, a new, unique record-id without qualifiers + * (if qualifiers not easily implemented using passed URI scheme). + * @throws URISyntaxException + */ + public URI getQualifiedRecordID(final String key, final String value) + throws URISyntaxException; + + /** + * Append (or if already present, update) qualifiers to passed + * recordId. Use with caution. Guard against turning up a + * result that already exists. Use when writing a group of records inside + * a single transaction. + * + * How qualifiers are appended/updated varies with URI scheme. Its allowed + * that an invocation of this method does nought but call + * {@link #getRecordID()}, returning a new URI unrelated to the passed + * recordId and passed qualifier. + * @param recordId URI to append qualifier to. + * @param qualifiers Map of qualifier values keyed by qualifier name. + * @return New URI based off passed uri and passed qualifier. + * @throws URISyntaxException if probably constructing URI OR if the + * resultant UUID does not differ from the one passed. + */ + public URI qualifyRecordID(final URI recordId, + final Map qualifiers) + throws URISyntaxException; +} diff --git a/commons/src/main/java/org/archive/uid/GeneratorFactory.java b/commons/src/main/java/org/archive/uid/GeneratorFactory.java new file mode 100644 index 00000000..32c882dd --- /dev/null +++ b/commons/src/main/java/org/archive/uid/GeneratorFactory.java @@ -0,0 +1,79 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.uid; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Map; + +/** + * Factory that generates uids. + * Singleton. Default implementation is {@link UUIDGenerator}. To + * change, specify alternate implementation of {@link Generator} with + * {@link #SYSTEM_PROPERTY_GENERATOR_KEY} system property. + * @author stack + * @version $Revision$ $Date$ + */ +public class GeneratorFactory implements Generator { + public final String SYSTEM_PROPERTY_GENERATOR_KEY = + this.getClass().toString() + ".generator"; + private static final String DEFAULT_GENERATOR = + "org.archive.uid.UUIDGenerator"; + private static final GeneratorFactory factory = new GeneratorFactory(); + private final Generator generator; + + private GeneratorFactory() { + super(); + String className = System.getProperty(SYSTEM_PROPERTY_GENERATOR_KEY, + DEFAULT_GENERATOR); + Generator ridg = null; + try { + Class c = Class.forName(className); + ridg = (Generator) c.newInstance(); + } catch (Exception e) { + e.printStackTrace(); + } + this.generator = ridg; + } + + public URI getRecordID() throws URISyntaxException { + return this.generator.getRecordID(); + } + + public URI getQualifiedRecordID(Map qualifiers) + throws URISyntaxException { + return this.generator.getQualifiedRecordID(qualifiers); + } + + public URI getQualifiedRecordID(String key, String value) + throws URISyntaxException { + return this.generator.getQualifiedRecordID(key, value); + } + + public URI qualifyRecordID(final URI uri, + final Map qualifiers) + throws URISyntaxException { + return this.generator.qualifyRecordID(uri, qualifiers); + } + + public static GeneratorFactory getFactory() { + return factory; + } +} diff --git a/commons/src/main/java/org/archive/uid/UUIDGenerator.java b/commons/src/main/java/org/archive/uid/UUIDGenerator.java new file mode 100644 index 00000000..d7809a98 --- /dev/null +++ b/commons/src/main/java/org/archive/uid/UUIDGenerator.java @@ -0,0 +1,74 @@ +/* $Id$ + * + * Created on July 27th, 2006 + * + * Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.uid; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Map; +import java.util.UUID; + +/** + * Generates UUIDs, using + * {@link java.util.UUID java.util.UUID}, formatted as URNs from the UUID + * namespace [See RFC4122]. + * Here is an examples of the type of ID it makes: + * urn:uuid:0161811f-5da6-4c6e-9808-a2fab97114cf. Always makes a + * new identifier even when passed qualifiers. + * + * @author stack + * @version $Revision$ $Date$ + * @see RFC4122 + */ +class UUIDGenerator implements Generator { + private static final String SCHEME = "urn:uuid"; + private static final String SCHEME_COLON = SCHEME + ":"; + + UUIDGenerator() { + super(); + } + + public synchronized URI qualifyRecordID(URI recordId, + final Map qualifiers) + throws URISyntaxException { + return getRecordID(); + } + + private String getUUID() { + return UUID.randomUUID().toString(); + } + + public URI getRecordID() throws URISyntaxException { + return new URI(SCHEME_COLON + getUUID()); + } + + public URI getQualifiedRecordID( + final String key, final String value) + throws URISyntaxException { + return getRecordID(); + } + + public URI getQualifiedRecordID(Map qualifiers) + throws URISyntaxException { + return getRecordID(); + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/uid/package.html b/commons/src/main/java/org/archive/uid/package.html new file mode 100644 index 00000000..dc49f07b --- /dev/null +++ b/commons/src/main/java/org/archive/uid/package.html @@ -0,0 +1,28 @@ + + + +org.archive.uid package + + +A unique ID generator. +Default is {@link org.archive.uid.UUIDGenerator}. +To use another ID Generator, set the System Property +org.archive.uid.GeneratorFactory.generator to point +at an alternate implementation of {@link org.archive.uid.Generator}. + +

TODO

+
    +
  • MIME boundaries have upper-bound of 70 characters total including + 'blank line' (CRLFCRLF) and two leading hyphens. Add to + {@link org.archive.uid.Generator} + interface an upper-bound on generated ID length.
    • +
  • Add example of an actionable uid generator: +e.g. http://archive.org/UID-SCHEME/ID +where scheme might be UUID and an ID might be +f9472055-fbb6-4810-90e8-68fd39e145a6;type=metadata or, +using ARK: +http://archive.org/ark:/13030/f9472055-fbb6-4810-90e8-68fd39e145a6;type=metadata. +
    • +
+ + diff --git a/commons/src/main/java/org/archive/util/AbstractLongFPSet.java b/commons/src/main/java/org/archive/util/AbstractLongFPSet.java new file mode 100644 index 00000000..7d45cdf1 --- /dev/null +++ b/commons/src/main/java/org/archive/util/AbstractLongFPSet.java @@ -0,0 +1,315 @@ +/* AbstractLongFPSet + * + * $Id$ + * + * Created on Oct 20, 2003 + * + * Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.util; + +import java.io.Serializable; +import java.util.logging.Logger; + +import org.archive.util.fingerprint.LongFPSet; + +/** + * Shell of functionality for a Set of primitive long fingerprints, held + * in an array of possibly-empty slots. + * + * The implementation of that holding array is delegated to subclasses. + * + *

Capacity is always a power of 2. + * + *

Fingerprints are already assumed to be well-distributed, so the + * hashed position for a value is just its high-order bits. + * + * @author gojomo + * @version $Date$, $Revision$ + */ +public abstract class AbstractLongFPSet implements LongFPSet, Serializable { + private static Logger logger = + Logger.getLogger("org.archive.util.AbstractLongFPSet"); + + /** + * A constant used to indicate that a slot in the set storage is empty. + * A zero or positive value means slot is filled + */ + protected static byte EMPTY = -1; + + /** the capacity of this set, specified as the exponent of a power of 2 */ + protected int capacityPowerOfTwo; + + /** The load factor, as a fraction. This gives the amount of free space + * to keep in the Set. */ + protected float loadFactor; + + /** The current number of elements in the set */ + protected long count; + + /** + * To support serialization + * TODO: verify needed? + */ + public AbstractLongFPSet() { + super(); + } + + /** + * Create a new AbstractLongFPSet with a given capacity and load Factor + * + * @param capacityPowerOfTwo The capacity as the exponent of a power of 2. + * e.g if the capacity is 4 this means 2^^4 + * entries + * @param loadFactor The load factor as a fraction. This gives the amount + * of free space to keep in the Set. + */ + public AbstractLongFPSet(final int capacityPowerOfTwo, float loadFactor) { + this.capacityPowerOfTwo = capacityPowerOfTwo; + this.loadFactor = loadFactor; + this.count = 0; + } + + /** + * Does this set contain the given value? + * + * @see org.archive.util.fingerprint.LongFPSet#contains(long) + */ + public boolean contains(long val) { + long i = indexFor(val); + if (slotHasData(i)) { + noteAccess(i); + return true; + } + return false; + } + + /** + * Check the state of a slot in the storage. + * + * @param i the index of the slot to check + * @return -1 if slot is filled; nonegative if full. + */ + protected abstract int getSlotState(long i); + + /** + * Note access (hook for subclass cache-replacement strategies) + * + * @param index The index of the slot to check. + */ + private void noteAccess(long index) { + // by default do nothing + // cache subclasses may use to update access counts, etc. + } + + /** + * Return the number of entries in this set. + * + * @see org.archive.util.fingerprint.LongFPSet#count() + */ + public long count() { + return count; + } + + /** + * Add the given value to this set + * + * @see org.archive.util.fingerprint.LongFPSet#add(long) + */ + public boolean add(long val) { + logger.finest("Adding " + val); + long i = indexFor(val); + if (slotHasData(i)) { + // positive index indicates already in set + return false; + } + // we have a possible slot now, which is encoded as a negative number + + // check for space, and grow if needed + if ((count + 1) > (loadFactor * (1 << capacityPowerOfTwo))) { + makeSpace(); + // find new i + i = indexFor(val); + assert i < 0 : "slot should be empty"; + } + + i = asDataSlot(i); // convert to positive index + setAt(i, val); + count++; + noteAccess(i); + return true; + } + + /** + * Make additional space to keep the load under the target + * loadFactor level. + * + * Subclasses may grow or discard entries to satisfy. + */ + protected abstract void makeSpace(); + + /** + * Set the stored value at the given slot. + * + * @param i the slot index + * @param l the value to set + */ + protected abstract void setAt(long i, long l); + + /** + * Get the stored value at the given slot. + * + * @param i the slot index + * @return The stored value at the given slot. + */ + protected abstract long getAt(long i); + + /** + * Given a value, check the store for its existence. If it exists, it + * will return the index where the value resides. Otherwise it return + * an encoded index, which is a possible storage location for the value. + * + *

Note, if we have a loading factor less than 1.0, there should always + * be an empty location where we can store the value + * + * @param val the fingerprint value to check for + * @return The (positive) index where the value already resides, + * or an empty index where it could be inserted (encoded as a + * negative number). + */ + private long indexFor(long val) { + long candidateIndex = startIndexFor(val); + while (true) { + if (getSlotState(candidateIndex) < 0) { + // slot empty; return negative number encoding index + return asEmptySlot(candidateIndex); + } + if (getAt(candidateIndex) == val) { + // already present; return positive index + return candidateIndex; + } + candidateIndex++; + if (candidateIndex == 1 << capacityPowerOfTwo) { + candidateIndex = 0; // wraparound + } + } + } + + /** + * Return the recommended storage index for the given value. + * Assumes values are already well-distributed; merely uses + * high-order bits. + * + * @param val + * @return The recommended storage index for the given value. + */ + private long startIndexFor(long val) { + return (val >>> (64 - capacityPowerOfTwo)); + } + + public boolean remove(long l) { + long i = indexFor(l); + if (!slotHasData(i)) { + // not present, not changed + return false; + } + removeAt(i); + return true; + } + + /** + * Remove the value at the given index, relocating its + * successors as necessary. + * + * @param index + */ + protected void removeAt(long index) { + count--; + clearAt(index); + long probeIndex = index + 1; + while (true) { + if (probeIndex == 1 << capacityPowerOfTwo) { + probeIndex = 0; //wraparound + } + if (getSlotState(probeIndex) < 0) { + // vacant + break; + } + long val = getAt(probeIndex); + long newIndex = indexFor(val); + if (newIndex != probeIndex) { + // value must shift down + newIndex = asDataSlot(newIndex); // positivize + relocate(val, probeIndex, newIndex); + } + probeIndex++; + } + } + + protected abstract void clearAt(long index); + + protected abstract void relocate(long value, long fromIndex, long toIndex); + + /** + * Low-cost, non-definitive (except when true) contains + * test. Default answer of false is acceptable. + * + * @see org.archive.util.fingerprint.LongFPSet#quickContains(long) + */ + public boolean quickContains(long fp) { + return false; + } + + /** + * given a slot index, which could or could not be empty, return it as + * a slot index indicating an non-empty slot + * + * @param index the slot index to convert + * @return the index, converted to represent an slot with data + */ + private long asDataSlot(final long index) { + if (slotHasData(index)) { // slot already has data + return index; + } + return - (index + 1); + } + + /** + * Given a slot index, which could or could not be empty, return it as + * a slot index indicating an empty slot + * @param index the slot index to convert + * @return the index, converted to represent an empty slot + */ + private long asEmptySlot(final long index) { + if (!slotHasData(index)) { // already empty slot + return index; + } + return -index - 1; + } + + /** + * Does this index represent a slot with data? + * + * @param index the index to check + * @return true if the slot has data + */ + private boolean slotHasData(final long index) { + return index >= 0; + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/util/ArchiveUtils.java b/commons/src/main/java/org/archive/util/ArchiveUtils.java new file mode 100644 index 00000000..4d67084d --- /dev/null +++ b/commons/src/main/java/org/archive/util/ArchiveUtils.java @@ -0,0 +1,846 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.util; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.text.NumberFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.Locale; +import java.util.Map; +import java.util.TimeZone; + + +/** + * Miscellaneous useful methods. + * + * @author gojomo & others + */ +public class ArchiveUtils { + + + final public static String VERSION = loadVersion(); + + /** + * Arc-style date stamp in the format yyyyMMddHHmm and UTC time zone. + */ + private static final ThreadLocal + TIMESTAMP12 = threadLocalDateFormat("yyyyMMddHHmm");; + + /** + * Arc-style date stamp in the format yyyyMMddHHmmss and UTC time zone. + */ + private static final ThreadLocal + TIMESTAMP14 = threadLocalDateFormat("yyyyMMddHHmmss"); + /** + * Arc-style date stamp in the format yyyyMMddHHmmssSSS and UTC time zone. + */ + private static final ThreadLocal + TIMESTAMP17 = threadLocalDateFormat("yyyyMMddHHmmssSSS"); + + /** + * Log-style date stamp in the format yyyy-MM-dd'T'HH:mm:ss.SSS'Z' + * UTC time zone is assumed. + */ + private static final ThreadLocal + TIMESTAMP17ISO8601Z = threadLocalDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + + /** + * Log-style date stamp in the format yyyy-MM-dd'T'HH:mm:ss'Z' + * UTC time zone is assumed. + */ + private static final ThreadLocal + TIMESTAMP14ISO8601Z = threadLocalDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); + + /** + * Default character to use padding strings. + */ + private static final char DEFAULT_PAD_CHAR = ' '; + + /** milliseconds in an hour */ + private static final int HOUR_IN_MS = 60 * 60 * 1000; + /** milliseconds in a day */ + private static final int DAY_IN_MS = 24 * HOUR_IN_MS; + + private static ThreadLocal threadLocalDateFormat(final String pattern) { + ThreadLocal tl = new ThreadLocal() { + protected SimpleDateFormat initialValue() { + SimpleDateFormat df = new SimpleDateFormat(pattern); + df.setTimeZone(TimeZone.getTimeZone("GMT")); + return df; + } + }; + return tl; + } + + public static int MAX_INT_CHAR_WIDTH = + Integer.toString(Integer.MAX_VALUE).length(); + + /** + * Utility function for creating arc-style date stamps + * in the format yyyMMddHHmmssSSS. + * Date stamps are in the UTC time zone + * @return the date stamp + */ + public static String get17DigitDate(){ + return TIMESTAMP17.get().format(new Date()); + } + + /** + * Utility function for creating arc-style date stamps + * in the format yyyMMddHHmmss. + * Date stamps are in the UTC time zone + * @return the date stamp + */ + public static String get14DigitDate(){ + return TIMESTAMP14.get().format(new Date()); + } + + /** + * Utility function for creating arc-style date stamps + * in the format yyyMMddHHmm. + * Date stamps are in the UTC time zone + * @return the date stamp + */ + public static String get12DigitDate(){ + return TIMESTAMP12.get().format(new Date()); + } + + /** + * Utility function for creating log timestamps, in + * W3C/ISO8601 format, assuming UTC. Use current time. + * + * Format is yyyy-MM-dd'T'HH:mm:ss.SSS'Z' + * + * @return the date stamp + */ + public static String getLog17Date(){ + return TIMESTAMP17ISO8601Z.get().format(new Date()); + } + + /** + * Utility function for creating log timestamps, in + * W3C/ISO8601 format, assuming UTC. + * + * Format is yyyy-MM-dd'T'HH:mm:ss.SSS'Z' + * @param date Date to format. + * + * @return the date stamp + */ + public static String getLog17Date(long date){ + return TIMESTAMP17ISO8601Z.get().format(new Date(date)); + } + + /** + * Utility function for creating log timestamps, in + * W3C/ISO8601 format, assuming UTC. Use current time. + * + * Format is yyyy-MM-dd'T'HH:mm:ss'Z' + * + * @return the date stamp + */ + public static String getLog14Date(){ + return TIMESTAMP14ISO8601Z.get().format(new Date()); + } + + /** + * Utility function for creating log timestamps, in + * W3C/ISO8601 format, assuming UTC. + * + * Format is yyyy-MM-dd'T'HH:mm:ss'Z' + * @param date long timestamp to format. + * + * @return the date stamp + */ + public static String getLog14Date(long date){ + return TIMESTAMP14ISO8601Z.get().format(new Date(date)); + } + + /** + * Utility function for creating log timestamps, in + * W3C/ISO8601 format, assuming UTC. + * + * Format is yyyy-MM-dd'T'HH:mm:ss'Z' + * @param date Date to format. + * + * @return the date stamp + */ + public static String getLog14Date(Date date){ + return TIMESTAMP14ISO8601Z.get().format(date); + } + + /** + * Utility function for creating arc-style date stamps + * in the format yyyyMMddHHmmssSSS. + * Date stamps are in the UTC time zone + * + * @param date milliseconds since epoc + * @return the date stamp + */ + public static String get17DigitDate(long date){ + return TIMESTAMP17.get().format(new Date(date)); + } + + public static String get17DigitDate(Date date){ + return TIMESTAMP17.get().format(date); + } + + /** + * Utility function for creating arc-style date stamps + * in the format yyyyMMddHHmmss. + * Date stamps are in the UTC time zone + * + * @param date milliseconds since epoc + * @return the date stamp + */ + public static String get14DigitDate(long date){ + return TIMESTAMP14.get().format(new Date(date)); + } + + public static String get14DigitDate(Date d) { + return TIMESTAMP14.get().format(d); + } + + /** + * Utility function for creating arc-style date stamps + * in the format yyyyMMddHHmm. + * Date stamps are in the UTC time zone + * + * @param date milliseconds since epoc + * @return the date stamp + */ + public static String get12DigitDate(long date){ + return TIMESTAMP12.get().format(new Date(date)); + } + + public static String get12DigitDate(Date d) { + return TIMESTAMP12.get().format(d); + } + + /** + * Parses an ARC-style date. If passed String is < 12 characters in length, + * we pad. At a minimum, String should contain a year (>=4 characters). + * Parse will also fail if day or month are incompletely specified. Depends + * on the above getXXDigitDate methods. + * @param A 4-17 digit date in ARC style (yyyy to + * yyyyMMddHHmmssSSS) formatting. + * @return A Date object representing the passed String. + * @throws ParseException + */ + public static Date getDate(String d) throws ParseException { + Date date = null; + if (d == null) { + throw new IllegalArgumentException("Passed date is null"); + } + switch (d.length()) { + case 14: + date = ArchiveUtils.parse14DigitDate(d); + break; + + case 17: + date = ArchiveUtils.parse17DigitDate(d); + break; + + case 12: + date = ArchiveUtils.parse12DigitDate(d); + break; + + case 0: + case 1: + case 2: + case 3: + throw new ParseException("Date string must at least contain a" + + "year: " + d, d.length()); + + default: + if (!(d.startsWith("19") || d.startsWith("20"))) { + throw new ParseException("Unrecognized century: " + d, 0); + } + if (d.length() < 8 && (d.length() % 2) != 0) { + throw new ParseException("Incomplete month/date: " + d, + d.length()); + } + StringBuilder sb = new StringBuilder(d); + if (sb.length() < 8) { + for (int i = sb.length(); sb.length() < 8; i += 2) { + sb.append("01"); + } + } + if (sb.length() < 12) { + for (int i = sb.length(); sb.length() < 12; i++) { + sb.append("0"); + } + } + date = ArchiveUtils.parse12DigitDate(sb.toString()); + } + + return date; + } + + /** + * Utility function for parsing arc-style date stamps + * in the format yyyMMddHHmmssSSS. + * Date stamps are in the UTC time zone. The whole string will not be + * parsed, only the first 17 digits. + * + * @param date an arc-style formatted date stamp + * @return the Date corresponding to the date stamp string + * @throws ParseException if the inputstring was malformed + */ + public static Date parse17DigitDate(String date) throws ParseException { + return TIMESTAMP17.get().parse(date); + } + + /** + * Utility function for parsing arc-style date stamps + * in the format yyyMMddHHmmss. + * Date stamps are in the UTC time zone. The whole string will not be + * parsed, only the first 14 digits. + * + * @param date an arc-style formatted date stamp + * @return the Date corresponding to the date stamp string + * @throws ParseException if the inputstring was malformed + */ + public static Date parse14DigitDate(String date) throws ParseException{ + return TIMESTAMP14.get().parse(date); + } + + /** + * Utility function for parsing arc-style date stamps + * in the format yyyMMddHHmm. + * Date stamps are in the UTC time zone. The whole string will not be + * parsed, only the first 12 digits. + * + * @param date an arc-style formatted date stamp + * @return the Date corresponding to the date stamp string + * @throws ParseException if the inputstring was malformed + */ + public static Date parse12DigitDate(String date) throws ParseException{ + return TIMESTAMP12.get().parse(date); + } + + /** + * Convert 17-digit date format timestamps (as found in crawl.log, for + * example) into a GregorianCalendar object. + * Useful so you can convert + * into milliseconds-since-epoch. Note: it is possible to compute + * milliseconds-since-epoch + * using {@link #parse17DigitDate}.UTC(), but + * that method is deprecated in favor of using Calendar.getTimeInMillis(). + * + *

I probably should have dug into all the utility methods in + * DateFormat.java to parse the timestamp, but this was + * easier. If + * someone wants to fix this to use those methods, please have at it!

+ * Mike Schwartz, schwartz at CodeOnTheRoad dot com. + * + * @param timestamp17String + * @return Calendar set to timestamp17String. + */ + public static Calendar timestamp17ToCalendar(String timestamp17String) { + GregorianCalendar calendar = new GregorianCalendar(); + int year = Integer.parseInt(timestamp17String.substring(0, 4)); + int dayOfMonth = Integer.parseInt(timestamp17String.substring(6, 8)); + // Month is 0-based + int month = Integer.parseInt(timestamp17String.substring(4, 6)) - 1; + int hourOfDay = Integer.parseInt(timestamp17String.substring(8, 10)); + int minute = Integer.parseInt(timestamp17String.substring(10, 12)); + int second = Integer.parseInt(timestamp17String.substring(12, 14)); + int milliseconds = Integer + .parseInt(timestamp17String.substring(14, 17)); + calendar.set(Calendar.YEAR, year); + calendar.set(Calendar.MONTH, month); + calendar.set(Calendar.DAY_OF_MONTH, dayOfMonth); + calendar.set(Calendar.HOUR_OF_DAY, hourOfDay); + calendar.set(Calendar.MINUTE, minute); + calendar.set(Calendar.SECOND, second); + calendar.set(Calendar.MILLISECOND, milliseconds); + return calendar; + } + + /** + * @param timestamp A 14-digit timestamp or the suffix for a 14-digit + * timestamp: E.g. '20010909014640' or '20010101' or '1970'. + * @return Seconds since the epoch as a string zero-pre-padded so always + * Integer.MAX_VALUE wide (Makes it so sorting of resultant string works + * properly). + * @throws ParseException + */ + public static String secondsSinceEpoch(String timestamp) + throws ParseException { + return zeroPadInteger((int) + (getSecondsSinceEpoch(timestamp).getTime()/1000)); + } + + /** + * @param timestamp A 14-digit timestamp or the suffix for a 14-digit + * timestamp: E.g. '20010909014640' or '20010101' or '1970'. + * @return A date. + * @see #secondsSinceEpoch(String) + * @throws ParseException + */ + public static Date getSecondsSinceEpoch(String timestamp) + throws ParseException { + if (timestamp.length() < 14) { + if (timestamp.length() < 10 && (timestamp.length() % 2) == 1) { + throw new IllegalArgumentException("Must have year, " + + "month, date, hour or second granularity: " + timestamp); + } + if (timestamp.length() == 4) { + // Add first month and first date. + timestamp = timestamp + "01010000"; + } + if (timestamp.length() == 6) { + // Add a date of the first. + timestamp = timestamp + "010000"; + } + if (timestamp.length() < 14) { + timestamp = timestamp + + ArchiveUtils.padTo("", 14 - timestamp.length(), '0'); + } + } + return ArchiveUtils.parse14DigitDate(timestamp); + } + + /** + * @param i Integer to add prefix of zeros too. If passed + * 2005, will return the String 0000002005. String + * width is the width of Integer.MAX_VALUE as a string (10 + * digits). + * @return Padded String version of i. + */ + public static String zeroPadInteger(int i) { + return ArchiveUtils.padTo(Integer.toString(i), + MAX_INT_CHAR_WIDTH, '0'); + } + + /** + * Convert an int to a String, and pad it to + * pad spaces. + * @param i the int + * @param pad the width to pad to. + * @return String w/ padding. + */ + public static String padTo(final int i, final int pad) { + String n = Integer.toString(i); + return padTo(n, pad); + } + + /** + * Pad the given String to pad characters wide + * by pre-pending spaces. s should not be null. + * If s is already wider than pad no change is + * done. + * + * @param s the String to pad + * @param pad the width to pad to. + * @return String w/ padding. + */ + public static String padTo(final String s, final int pad) { + return padTo(s, pad, DEFAULT_PAD_CHAR); + } + + /** + * Pad the given String to pad characters wide + * by pre-pending padChar. + * + * s should not be null. If s is + * already wider than pad no change is done. + * + * @param s the String to pad + * @param pad the width to pad to. + * @param padChar The pad character to use. + * @return String w/ padding. + */ + public static String padTo(final String s, final int pad, + final char padChar) { + String result = s; + int l = s.length(); + if (l < pad) { + StringBuffer sb = new StringBuffer(pad); + while(l < pad) { + sb.append(padChar); + l++; + } + sb.append(s); + result = sb.toString(); + } + return result; + } + + /** check that two byte arrays are equal. They may be null. + * + * @param lhs a byte array + * @param rhs another byte array. + * @return true if they are both equal (or both + * null) + */ + public static boolean byteArrayEquals(final byte[] lhs, final byte[] rhs) { + if (lhs == null && rhs != null || lhs != null && rhs == null) { + return false; + } + if (lhs==rhs) { + return true; + } + if (lhs.length != rhs.length) { + return false; + } + for(int i = 0; i + * This involves converting it to the largest unit + * (of B, KB, MB, GB, TB) for which the amount will be > 1. + *

+ * Additionally, at least 2 significant digits are always displayed. + *

+ * Displays as bytes (B): 0-1023 + * Displays as kilobytes (KB): 1024 - 2097151 (~2Mb) + * Displays as megabytes (MB): 2097152 - 4294967295 (~4Gb) + * Displays as gigabytes (GB): 4294967296 - infinity + *

+ * Negative numbers will be returned as '0 B'. + * + * @param amount the amount of bytes + * @return A string containing the amount, properly formated. + */ + public static String formatBytesForDisplay(long amount) { + double displayAmount = (double) amount; + int unitPowerOf1024 = 0; + + if(amount <= 0){ + return "0 B"; + } + + while(displayAmount>=1024 && unitPowerOf1024 < 4) { + displayAmount = displayAmount / 1024; + unitPowerOf1024++; + } + + // TODO: get didactic, make these KiB, MiB, GiB, TiB + final String[] units = { " B", " KB", " MB", " GB", " TB" }; + + // ensure at least 2 significant digits (#.#) for small displayValues + int fractionDigits = (displayAmount < 10) ? 1 : 0; + return doubleToString(displayAmount, fractionDigits, fractionDigits) + + units[unitPowerOf1024]; + } + + /** + * Convert milliseconds value to a human-readable duration + * @param time + * @return Human readable string version of passed time + */ + public static String formatMillisecondsToConventional(long time) { + return formatMillisecondsToConventional(time,true); + } + + /** + * Convert milliseconds value to a human-readable duration + * @param time + * @param toMs whether to print to the ms + * @return Human readable string version of passed time + */ + public static String formatMillisecondsToConventional(long time, boolean toMs) { + return formatMillisecondsToConventional(time,5); + } + + /** + * Convert milliseconds value to a human-readable duration of + * mixed units, using units no larger than days. For example, + * "5d12h13m12s113ms" or "19h51m". + * + * @param duration + * @param unitCount how many significant units to show, at most + * for example, a value of 2 would show days+hours or hours+seconds + * but not hours+second+milliseconds + * @return Human readable string version of passed time + */ + public static String formatMillisecondsToConventional(long duration, int unitCount) { + if(unitCount <=0) { + unitCount = 5; + } + if(duration==0) { + return "0ms"; + } + StringBuffer sb = new StringBuffer(); + if(duration<0) { + sb.append("-"); + } + long absTime = Math.abs(duration); + long[] thresholds = {DAY_IN_MS, HOUR_IN_MS, 60000, 1000, 1}; + String[] units = {"d","h","m","s","ms"}; + + for(int i = 0; i < thresholds.length; i++) { + if(absTime >= thresholds[i]) { + sb.append(absTime / thresholds[i] + units[i]); + absTime = absTime % thresholds[i]; + unitCount--; + } + if(unitCount==0) { + break; + } + } + return sb.toString(); + } + + /** + * Copy the raw bytes of a long into a byte array, starting at + * the specified offset. + * + * @param l + * @param array + * @param offset + */ + public static void longIntoByteArray(long l, byte[] array, int offset) { + int i, shift; + + for(i = 0, shift = 56; i < 8; i++, shift -= 8) + array[offset+i] = (byte)(0xFF & (l >> shift)); + } + + public static long byteArrayIntoLong(byte [] bytearray) { + return byteArrayIntoLong(bytearray, 0); + } + + /** + * Byte array into long. + * @param bytearray Array to convert to a long. + * @param offset Offset into array at which we start decoding the long. + * @return Long made of the bytes of array beginning at + * offset offset. + * @see #longIntoByteArray(long, byte[], int) + */ + public static long byteArrayIntoLong(byte [] bytearray, + int offset) { + long result = 0; + for (int i = offset; i < 8 /*Bytes in long*/; i++) { + result = (result << 8 /*Bits in byte*/) | + (0xff & (byte)(bytearray[i] & 0xff)); + } + return result; + } + + /** + * Given a string that may be a plain host or host/path (without + * URI scheme), add an implied http:// if necessary. + * + * @param u string to evaluate + * @return string with http:// added if no scheme already present + */ + public static String addImpliedHttpIfNecessary(String u) { + int colon = u.indexOf(':'); + int period = u.indexOf('.'); + if (colon == -1 || (period >= 0) && (period < colon)) { + // No scheme present; prepend "http://" + u = "http://" + u; + } + return u; + } + + /** + * Verify that the array begins with the prefix. + * + * @param array + * @param prefix + * @return true if array is identical to prefix for the first prefix.length + * positions + */ + public static boolean startsWith(byte[] array, byte[] prefix) { + if(prefix.length>array.length) { + return false; + } + for(int i = 0; i < prefix.length; i++) { + if(array[i]!=prefix[i]) { + return false; + } + } + return true; + } + + /** + * Utility method to get a String singleLineReport from Reporter + * @param rep Reporter to get singleLineReport from + * @return String of report + */ + public static String singleLineReport(Reporter rep) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + try { + rep.singleLineReportTo(pw); + } catch (IOException e) { + // not really possible + e.printStackTrace(); + } + pw.flush(); + return sw.toString(); + } + + /** + * Compose the requested report into a String. DANGEROUS IF REPORT + * CAN BE LARGE. + * + * @param rep Reported + * @param name String name of report to compose + * @return String of report + */ + public static String writeReportToString(MultiReporter rep, String name) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + rep.reportTo(name,pw); + pw.flush(); + return sw.toString(); + } + + /** + * Enhance given object's default String display for appearing + * nested in a pretty Map String. + * + * @param obj Object to prettify + * @return prettified String + */ + public static String prettyString(Object obj) { + return "<"+obj+">"; + } + + /** + * Provide a improved String of a Map's entries + * + * @param Map + * @return prettified (in curly brackets) string of Map contents + */ + @SuppressWarnings("unchecked") + public static String prettyString(Map map) { + StringBuilder builder = new StringBuilder(); + builder.append("{ "); + boolean needsComma = false; + for( Object key : map.keySet()) { + if(needsComma) { + builder.append(", "); + } + builder.append(key); + builder.append(": "); + builder.append(prettyString(map.get(key))); + needsComma = true; + } + builder.append(" }"); + return builder.toString(); + } + + /** + * Provide a slightly-improved String of Map[] + * + * @param Map[] + * @return prettified (in square brackets) of Map[] + */ + @SuppressWarnings("unchecked") + public static String prettyString(Map[] maps) { + StringBuilder builder = new StringBuilder(); + builder.append("[ "); + boolean needsComma = false; + for( Map map : maps) { + if(map==null) continue; + if(needsComma) { + builder.append(", "); + } + builder.append(prettyString(map)); + needsComma = true; + } + builder.append(" ]"); + return builder.toString(); + } + + + private static String loadVersion() { + InputStream input = ArchiveUtils.class.getResourceAsStream( + "/org/archive/util/version.txt"); + if (input == null) { + return "UNKNOWN"; + } + BufferedReader br = null; + String version; + try { + br = new BufferedReader(new InputStreamReader(input)); + version = br.readLine(); + br.readLine(); + } catch (IOException e) { + return e.getMessage(); + } finally { + IoUtils.close(br); + } + + version = version.trim(); + if (!version.endsWith("SNAPSHOT")) { + return version; + } + + input = ArchiveUtils.class.getResourceAsStream("/org/archive/util/timestamp.txt"); + if (input == null) { + return version; + } + + br = null; + String timestamp; + try { + br = new BufferedReader(new InputStreamReader(input)); + timestamp = br.readLine(); + } catch (IOException e) { + return version; + } finally { + IoUtils.close(br); + } + + if (timestamp.startsWith("timestamp=")) { + timestamp = timestamp.substring(10); + } + + return version.trim() + "-" + timestamp.trim(); + } + +} + diff --git a/commons/src/main/java/org/archive/util/Base32.java b/commons/src/main/java/org/archive/util/Base32.java new file mode 100644 index 00000000..47d5bc86 --- /dev/null +++ b/commons/src/main/java/org/archive/util/Base32.java @@ -0,0 +1,165 @@ +/* Base32 +* +* $Id$ +* +* Created on Jan 21, 2004 +* +* Copyright (C) 2004 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util; + +/** + * Base32 - encodes and decodes RFC3548 Base32 + * (see http://www.faqs.org/rfcs/rfc3548.html ) + * + * Imported public-domain code of Bitzi. + * + * @author Robert Kaye + * @author Gordon Mohr + */ +public class Base32 { + private static final String base32Chars = + "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; + private static final int[] base32Lookup = + { 0xFF,0xFF,0x1A,0x1B,0x1C,0x1D,0x1E,0x1F, // '0', '1', '2', '3', '4', '5', '6', '7' + 0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF, // '8', '9', ':', ';', '<', '=', '>', '?' + 0xFF,0x00,0x01,0x02,0x03,0x04,0x05,0x06, // '@', 'A', 'B', 'C', 'D', 'E', 'F', 'G' + 0x07,0x08,0x09,0x0A,0x0B,0x0C,0x0D,0x0E, // 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O' + 0x0F,0x10,0x11,0x12,0x13,0x14,0x15,0x16, // 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W' + 0x17,0x18,0x19,0xFF,0xFF,0xFF,0xFF,0xFF, // 'X', 'Y', 'Z', '[', '\', ']', '^', '_' + 0xFF,0x00,0x01,0x02,0x03,0x04,0x05,0x06, // '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g' + 0x07,0x08,0x09,0x0A,0x0B,0x0C,0x0D,0x0E, // 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o' + 0x0F,0x10,0x11,0x12,0x13,0x14,0x15,0x16, // 'p', 'q', 'r', 's', 't', 'u', 'v', 'w' + 0x17,0x18,0x19,0xFF,0xFF,0xFF,0xFF,0xFF // 'x', 'y', 'z', '{', '|', '}', '~', 'DEL' + }; + + /** + * Encodes byte array to Base32 String. + * + * @param bytes Bytes to encode. + * @return Encoded byte array bytes as a String. + * + */ + static public String encode(final byte[] bytes) { + int i = 0, index = 0, digit = 0; + int currByte, nextByte; + StringBuffer base32 = new StringBuffer((bytes.length + 7) * 8 / 5); + + while (i < bytes.length) { + currByte = (bytes[i] >= 0) ? bytes[i] : (bytes[i] + 256); // unsign + + /* Is the current digit going to span a byte boundary? */ + if (index > 3) { + if ((i + 1) < bytes.length) { + nextByte = + (bytes[i + 1] >= 0) ? bytes[i + 1] : (bytes[i + 1] + 256); + } else { + nextByte = 0; + } + + digit = currByte & (0xFF >> index); + index = (index + 5) % 8; + digit <<= index; + digit |= nextByte >> (8 - index); + i++; + } else { + digit = (currByte >> (8 - (index + 5))) & 0x1F; + index = (index + 5) % 8; + if (index == 0) + i++; + } + base32.append(base32Chars.charAt(digit)); + } + + return base32.toString(); + } + + /** + * Decodes the given Base32 String to a raw byte array. + * + * @param base32 + * @return Decoded base32 String as a raw byte array. + */ + static public byte[] decode(final String base32) { + int i, index, lookup, offset, digit; + byte[] bytes = new byte[base32.length() * 5 / 8]; + + for (i = 0, index = 0, offset = 0; i < base32.length(); i++) { + lookup = base32.charAt(i) - '0'; + + /* Skip chars outside the lookup table */ + if (lookup < 0 || lookup >= base32Lookup.length) { + continue; + } + + digit = base32Lookup[lookup]; + + /* If this digit is not in the table, ignore it */ + if (digit == 0xFF) { + continue; + } + + if (index <= 3) { + index = (index + 5) % 8; + if (index == 0) { + bytes[offset] |= digit; + offset++; + if (offset >= bytes.length) + break; + } else { + bytes[offset] |= digit << (8 - index); + } + } else { + index = (index + 5) % 8; + bytes[offset] |= (digit >>> index); + offset++; + + if (offset >= bytes.length) { + break; + } + bytes[offset] |= digit << (8 - index); + } + } + return bytes; + } + + /** For testing, take a command-line argument in Base32, decode, print in hex, + * encode, print + * + * @param args + */ + static public void main(String[] args) { + if (args.length == 0) { + System.out.println("Supply a Base32-encoded argument."); + return; + } + System.out.println(" Original: " + args[0]); + byte[] decoded = Base32.decode(args[0]); + System.out.print(" Hex: "); + for (int i = 0; i < decoded.length; i++) { + int b = decoded[i]; + if (b < 0) { + b += 256; + } + System.out.print((Integer.toHexString(b + 256)).substring(1)); + } + System.out.println(); + System.out.println("Reencoded: " + Base32.encode(decoded)); + } +} diff --git a/commons/src/main/java/org/archive/util/BenchmarkBlooms.java b/commons/src/main/java/org/archive/util/BenchmarkBlooms.java new file mode 100644 index 00000000..535564f9 --- /dev/null +++ b/commons/src/main/java/org/archive/util/BenchmarkBlooms.java @@ -0,0 +1,105 @@ +/* BenchmarkBlooms +* +* $Id$ +* +* Created on Jun 30, 2005 +* +* Copyright (C) 2005 Internet Archive +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util; + +/** + * Simple benchmarking of different BloomFilter + * implementations. + * + * Take care when interpreting results; the effect of GC, + * dynamic compilation, and any other activity on test + * machine may affect relative time tallies in unpredictable + * ways. + * + * @author Gordon Mohr + */ +public class BenchmarkBlooms { + + public static void main(String[] args) { + (new BenchmarkBlooms()).instanceMain(args); + } + + public void instanceMain(String[] args) { + int reps = + (args.length > 0) ? Integer.parseInt(args[0]) : 3; + int n_expected = + (args.length > 1) ? Integer.parseInt(args[1]) : 10000000; + int d_hashes = + (args.length > 2) ? Integer.parseInt(args[2]) : 22; + int adds = + (args.length > 3) ? Integer.parseInt(args[3]) : 5000000; + String prefix = + (args.length > 4) ? args[4] : "http://www.archive.org/"; + + System.out.println( + "reps="+reps+" n_expected="+n_expected+ + " d_hashes="+d_hashes+" adds="+adds+" prefix="+prefix); + + BloomFilter bloom64; + BloomFilter bloom32; + BloomFilter bloom32split; + BloomFilter bloom32p2; + BloomFilter bloom32p2split; + for (int r=0;rNote that this method may return true on a character sequence that is has + * not been added to the filter. This will happen with probability 2-d, + * where d is the number of hash functions specified at creation time, if + * the number of the elements in the filter is less than n, the number + * of expected elements specified at creation time. + * + * @param s a character sequence. + * @return true if the sequence is in the filter (or if a sequence with the + * same hash sequence is in the filter). + */ + public abstract boolean contains(final CharSequence s); + + /** Adds a character sequence to the filter. + * + * @param s a character sequence. + * @return true if the character sequence was not in the filter (but see {@link #contains(CharSequence)}). + */ + public abstract boolean add(final CharSequence s); + + /** + * The amount of memory in bytes consumed by the bloom + * bitfield. + * + * @return memory used by bloom bitfield, in bytes + */ + public abstract long getSizeBytes(); +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/util/BloomFilter32bit.java b/commons/src/main/java/org/archive/util/BloomFilter32bit.java new file mode 100644 index 00000000..7a52ce39 --- /dev/null +++ b/commons/src/main/java/org/archive/util/BloomFilter32bit.java @@ -0,0 +1,223 @@ +/* BloomFilter32bit +* +* $Id$ +* +* Created on Jun 21, 2005 +* +* Copyright (C) 2005 Internet Archive; a slight adaptation of +* LGPL work (C) Sebastiano Vigna +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +package org.archive.util; + +import java.io.Serializable; +import java.security.SecureRandom; + +/** A Bloom filter. + * + * SLIGHTLY ADAPTED VERSION OF MG4J it.unimi.dsi.mg4j.util.BloomFilter + * + *

KEY CHANGES: + * + *

    + *
  • Adapted to use 32bit ops as much as possible... may be slightly + * faster on 32bit hardware/OS
    • + *
  • NUMBER_OF_WEIGHTS is 2083, to better avoid collisions between + * similar strings
    • + *
  • Removed dependence on cern.colt MersenneTwister (replaced with + * SecureRandom) and QuickBitVector (replaced with local methods).
    • + *
+ * + *
+ * + *

Instances of this class represent a set of character sequences (with false positives) + * using a Bloom filter. Because of the way Bloom filters work, + * you cannot remove elements. + * + *

Bloom filters have an expected error rate, depending on the number + * of hash functions used, on the filter size and on the number of elements in the filter. This implementation + * uses a variable optimal number of hash functions, depending on the expected + * number of elements. More precisely, a Bloom + * filter for n character sequences with d hash functions will use + * ln 2 dn ≈ 1.44 dn bits; + * false positives will happen with probability 2-d. + * + *

Hash functions are generated at creation time using universal hashing. Each hash function + * uses {@link #NUMBER_OF_WEIGHTS} random integers, which are cyclically multiplied by + * the character codes in a character sequence. The resulting integers are XOR-ed together. + * + *

This class exports access methods that are very similar to those of {@link java.util.Set}, + * but it does not implement that interface, as too many non-optional methods + * would be unimplementable (e.g., iterators). + * + * @author Sebastiano Vigna + */ +public class BloomFilter32bit implements Serializable, BloomFilter { + + private static final long serialVersionUID = -1567837798979475689L; + + /** The number of weights used to create hash functions. */ + final public static int NUMBER_OF_WEIGHTS = 2083; // CHANGED FROM 16 + /** The number of bits in this filter. */ + final public long m; + /** The number of hash functions used by this filter. */ + final public int d; + /** The underlying bit vectorS. */ + final private int[] bits; + /** The random integers used to generate the hash functions. */ + final private int[][] weight; + + /** The number of elements currently in the filter. It may be + * smaller than the actual number of additions of distinct character + * sequences because of false positives. + */ + private int size; + + /** The natural logarithm of 2, used in the computation of the number of bits. */ + private final static double NATURAL_LOG_OF_2 = Math.log( 2 ); + + private final static boolean DEBUG = false; + + /** Creates a new Bloom filter with given number of hash functions and expected number of elements. + * + * @param n the expected number of elements. + * @param d the number of hash functions; if the filter add not more than n elements, + * false positives will happen with probability 2-d. + */ + public BloomFilter32bit( final int n, final int d ) { + this.d = d; + int len = + (int)Math.ceil( ( (long)n * (long)d / NATURAL_LOG_OF_2 ) / 32 ); + this.m = len*32L; + if ( m >= 1L<<32 ) { + throw new IllegalArgumentException( "This filter would require " + m + " bits" ); + } + bits = new int[ len ]; + + if ( DEBUG ) System.err.println( "Number of bits: " + m ); + + // seeded for reproduceable behavior in repeated runs; BUT: + // SecureRandom's default implementation (as of 1.5) + // seems to mix in its own seeding. + final SecureRandom random = new SecureRandom(new byte[] {19,96}); + weight = new int[ d ][]; + for( int i = 0; i < d; i++ ) { + weight[ i ] = new int[ NUMBER_OF_WEIGHTS ]; + for( int j = 0; j < NUMBER_OF_WEIGHTS; j++ ) + weight[ i ][ j ] = random.nextInt(); + } + } + + /** The number of character sequences in the filter. + * + * @return the number of character sequences in the filter (but see {@link #contains(CharSequence)}). + */ + + public int size() { + return size; + } + + /** Hashes the given sequence with the given hash function. + * + * @param s a character sequence. + * @param l the length of s. + * @param k a hash function index (smaller than {@link #d}). + * @return the position in the filter corresponding to s for the hash function k. + */ + private long hash( final CharSequence s, final int l, final int k ) { + final int[] w = weight[ k ]; + int h = 0, i = l; + while( i-- != 0 ) h ^= s.charAt( i ) * w[ i % NUMBER_OF_WEIGHTS ]; + return ((long)h-Integer.MIN_VALUE) % m; + } + + /** Checks whether the given character sequence is in this filter. + * + *

Note that this method may return true on a character sequence that is has + * not been added to the filter. This will happen with probability 2-d, + * where d is the number of hash functions specified at creation time, if + * the number of the elements in the filter is less than n, the number + * of expected elements specified at creation time. + * + * @param s a character sequence. + * @return true if the sequence is in the filter (or if a sequence with the + * same hash sequence is in the filter). + */ + + public boolean contains( final CharSequence s ) { + int i = d, l = s.length(); + while( i-- != 0 ) if ( ! getBit( hash( s, l, i ) ) ) return false; + return true; + } + + /** Adds a character sequence to the filter. + * + * @param s a character sequence. + * @return true if the character sequence was not in the filter (but see {@link #contains(CharSequence)}). + */ + + public boolean add( final CharSequence s ) { + boolean result = false; + int i = d, l = s.length(); + long h; + while( i-- != 0 ) { + h = hash( s, l, i ); + if ( ! getBit( h ) ) result = true; + setBit( h ); + } + if ( result ) size++; + return result; + } + + protected final static long ADDRESS_BITS_PER_UNIT = 5; // 32=2^5 + protected final static long BIT_INDEX_MASK = 31; // = BITS_PER_UNIT - 1; + + /** + * Returns from the local bitvector the value of the bit with + * the specified index. The value is true if the bit + * with the index bitIndex is currently set; otherwise, + * returns false. + * + * (adapted from cern.colt.bitvector.QuickBitVector) + * + * @param bitIndex the bit index. + * @return the value of the bit with the specified index. + */ + protected boolean getBit(long bitIndex) { + return ((bits[(int)(bitIndex >> ADDRESS_BITS_PER_UNIT)] & (1 << (bitIndex & BIT_INDEX_MASK))) != 0); + } + + /** + * Changes the bit with index bitIndex in local bitvector. + * + * (adapted from cern.colt.bitvector.QuickBitVector) + * + * @param bitIndex the index of the bit to be set. + */ + protected void setBit(long bitIndex) { + bits[(int)(bitIndex >> ADDRESS_BITS_PER_UNIT)] |= 1 << (bitIndex & BIT_INDEX_MASK); + } + + /* (non-Javadoc) + * @see org.archive.util.BloomFilter#getSizeBytes() + */ + public long getSizeBytes() { + return bits.length*4; + } +} diff --git a/commons/src/main/java/org/archive/util/BloomFilter32bitSplit.java b/commons/src/main/java/org/archive/util/BloomFilter32bitSplit.java new file mode 100644 index 00000000..fd71c884 --- /dev/null +++ b/commons/src/main/java/org/archive/util/BloomFilter32bitSplit.java @@ -0,0 +1,251 @@ +/* BloomFilter32bit +* +* $Id$ +* +* Created on Jun 21, 2005 +* +* Copyright (C) 2005 Internet Archive; a slight adaptation of +* LGPL work (C) Sebastiano Vigna +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +package org.archive.util; + +import java.io.Serializable; +import java.security.SecureRandom; + +/** A Bloom filter. + * + * SLIGHTLY ADAPTED VERSION OF MG4J it.unimi.dsi.mg4j.util.BloomFilter + * + *

KEY CHANGES: + * + *

    + *
  • Adapted to use 32bit ops as much as possible... may be slightly + * faster on 32bit hardware/OS
    • + *
  • NUMBER_OF_WEIGHTS is 2083, to better avoid collisions between + * similar strings
    • + *
  • Removed dependence on cern.colt MersenneTwister (replaced with + * SecureRandom) and QuickBitVector (replaced with local methods).
    • + *
+ * + *
+ * + *

Instances of this class represent a set of character sequences (with false positives) + * using a Bloom filter. Because of the way Bloom filters work, + * you cannot remove elements. + * + *

Bloom filters have an expected error rate, depending on the number + * of hash functions used, on the filter size and on the number of elements in the filter. This implementation + * uses a variable optimal number of hash functions, depending on the expected + * number of elements. More precisely, a Bloom + * filter for n character sequences with d hash functions will use + * ln 2 dn ≈ 1.44 dn bits; + * false positives will happen with probability 2-d. + * + *

Hash functions are generated at creation time using universal hashing. Each hash function + * uses {@link #NUMBER_OF_WEIGHTS} random integers, which are cyclically multiplied by + * the character codes in a character sequence. The resulting integers are XOR-ed together. + * + *

This class exports access methods that are very similar to those of {@link java.util.Set}, + * but it does not implement that interface, as too many non-optional methods + * would be unimplementable (e.g., iterators). + * + * @author Sebastiano Vigna + */ +public class BloomFilter32bitSplit implements Serializable, BloomFilter { + + private static final long serialVersionUID = -164106965277863971L; + + /** The number of weights used to create hash functions. */ + final public static int NUMBER_OF_WEIGHTS = 2083; // CHANGED FROM 16 + /** The number of bits in this filter. */ + final public long m; + /** The number of hash functions used by this filter. */ + final public int d; + /** The underlying bit vectorS. */ +// final private int[] bits; + final private int[][] bits; + /** The random integers used to generate the hash functions. */ + final private int[][] weight; + + /** The number of elements currently in the filter. It may be + * smaller than the actual number of additions of distinct character + * sequences because of false positives. + */ + private int size; + + /** The natural logarithm of 2, used in the computation of the number of bits. */ + private final static double NATURAL_LOG_OF_2 = Math.log( 2 ); + + /** number of ints in 1MB. */ + private final static int ONE_MB_INTS = 1 << 18; // + + private final static boolean DEBUG = false; + + /** Creates a new Bloom filter with given number of hash functions and expected number of elements. + * + * @param n the expected number of elements. + * @param d the number of hash functions; if the filter add not more than n elements, + * false positives will happen with probability 2-d. + */ + public BloomFilter32bitSplit( final int n, final int d ) { + this.d = d; + int len = + (int)Math.ceil( ( (long)n * (long)d / NATURAL_LOG_OF_2 ) / 32 ); + // round up to ensure divisible into 1MiB chunks + len = ((len / ONE_MB_INTS)+1)*ONE_MB_INTS; + this.m = len*32L; + if ( m >= 1L<<54 ) { + throw new IllegalArgumentException( "This filter would require " + m + " bits" ); + } +// bits = new int[ len ]; + bits = new int[ len/ONE_MB_INTS ][ONE_MB_INTS]; + + if ( DEBUG ) System.err.println( "Number of bits: " + m ); + + // seeded for reproduceable behavior in repeated runs; BUT: + // SecureRandom's default implementation (as of 1.5) + // seems to mix in its own seeding. + final SecureRandom random = new SecureRandom(new byte[] {19,96}); + weight = new int[ d ][]; + for( int i = 0; i < d; i++ ) { + weight[ i ] = new int[ NUMBER_OF_WEIGHTS ]; + for( int j = 0; j < NUMBER_OF_WEIGHTS; j++ ) + weight[ i ][ j ] = random.nextInt(); + } + } + + /** The number of character sequences in the filter. + * + * @return the number of character sequences in the filter (but see {@link #contains(CharSequence)}). + */ + + public int size() { + return size; + } + + /** Hashes the given sequence with the given hash function. + * + * @param s a character sequence. + * @param l the length of s. + * @param k a hash function index (smaller than {@link #d}). + * @return the position in the filter corresponding to s for the hash function k. + */ + private long hash( final CharSequence s, final int l, final int k ) { + final int[] w = weight[ k ]; + int h = 0, i = l; + while( i-- != 0 ) h ^= s.charAt( i ) * w[ i % NUMBER_OF_WEIGHTS ]; + return ((long)h-Integer.MIN_VALUE) % m; + } + + /** Checks whether the given character sequence is in this filter. + * + *

Note that this method may return true on a character sequence that is has + * not been added to the filter. This will happen with probability 2-d, + * where d is the number of hash functions specified at creation time, if + * the number of the elements in the filter is less than n, the number + * of expected elements specified at creation time. + * + * @param s a character sequence. + * @return true if the sequence is in the filter (or if a sequence with the + * same hash sequence is in the filter). + */ + + public boolean contains( final CharSequence s ) { + int i = d, l = s.length(); + while( i-- != 0 ) if ( ! getBit( hash( s, l, i ) ) ) return false; + return true; + } + + /** Adds a character sequence to the filter. + * + * @param s a character sequence. + * @return true if the character sequence was not in the filter (but see {@link #contains(CharSequence)}). + */ + + public boolean add( final CharSequence s ) { + boolean result = false; + int i = d, l = s.length(); + long h; + while( i-- != 0 ) { + h = hash( s, l, i ); + if ( ! setGetBit( h ) ) result = true; + } + if ( result ) size++; + return result; + } + + protected final static long ADDRESS_BITS_PER_UNIT = 5; // 32=2^5 + protected final static long BIT_INDEX_MASK = 31; // = BITS_PER_UNIT - 1; + + /** + * Returns from the local bitvector the value of the bit with + * the specified index. The value is true if the bit + * with the index bitIndex is currently set; otherwise, + * returns false. + * + * (adapted from cern.colt.bitvector.QuickBitVector) + * + * @param bitIndex the bit index. + * @return the value of the bit with the specified index. + */ + protected boolean getBit(long bitIndex) { + long intIndex = (bitIndex >>> ADDRESS_BITS_PER_UNIT); + return ((bits[(int)(intIndex / ONE_MB_INTS)][(int)(intIndex % ONE_MB_INTS)] + & (1 << (bitIndex & BIT_INDEX_MASK))) != 0); + } + + /** + * Changes the bit with index bitIndex in local bitvector. + * + * (adapted from cern.colt.bitvector.QuickBitVector) + * + * @param bitIndex the index of the bit to be set. + */ + protected void setBit(long bitIndex) { + long intIndex = (bitIndex >>> ADDRESS_BITS_PER_UNIT); + bits[(int)(intIndex / ONE_MB_INTS)][(int)(intIndex % ONE_MB_INTS)] + |= 1 << (bitIndex & BIT_INDEX_MASK); + } + + /** + * Sets the bit with index bitIndex in local bitvector -- + * returning the old value. + * + * (adapted from cern.colt.bitvector.QuickBitVector) + * + * @param bitIndex the index of the bit to be set. + */ + protected boolean setGetBit(long bitIndex) { + long intIndex = (int) (bitIndex >>> ADDRESS_BITS_PER_UNIT); + int a = (int)(intIndex / ONE_MB_INTS); + int b = (int)(intIndex % ONE_MB_INTS); + int mask = 1 << (bitIndex & BIT_INDEX_MASK); + boolean ret = ((bits[a][b] & (mask)) != 0); + bits[a][b] |= mask; + return ret; + } + + /* (non-Javadoc) + * @see org.archive.util.BloomFilter#getSizeBytes() + */ + public long getSizeBytes() { + return bits.length*bits[0].length*4; + } +} diff --git a/commons/src/main/java/org/archive/util/BloomFilter32bp2.java b/commons/src/main/java/org/archive/util/BloomFilter32bp2.java new file mode 100644 index 00000000..ffa64d66 --- /dev/null +++ b/commons/src/main/java/org/archive/util/BloomFilter32bp2.java @@ -0,0 +1,235 @@ +/* BloomFilter +* +* $Id$ +* +* Created on Jun 21, 2005 +* +* Copyright (C) 2005 Internet Archive; a slight adaptation of +* LGPL work (C) Sebastiano Vigna +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +package org.archive.util; + +import java.io.Serializable; +import java.security.SecureRandom; + +/** A Bloom filter. + * + * SLIGHTLY ADAPTED VERSION OF MG4J it.unimi.dsi.mg4j.util.BloomFilter + * + *

KEY CHANGES: + * + *

    + *
  • Adapted to use 32bit ops as much as possible... may be slightly + * faster on 32bit hardware/OS
    • + *
  • Changed to use bitfield that is a power-of-two in size, allowing + * hash() to use bitshifting rather than modulus... may be slightly + * faster
    • + *
  • NUMBER_OF_WEIGHTS is 2083, to better avoid collisions between + * similar strings
    • + *
  • Removed dependence on cern.colt MersenneTwister (replaced with + * SecureRandom) and QuickBitVector (replaced with local methods).
    • + *
+ * + *
+ * + *

Instances of this class represent a set of character sequences (with false positives) + * using a Bloom filter. Because of the way Bloom filters work, + * you cannot remove elements. + * + *

Bloom filters have an expected error rate, depending on the number + * of hash functions used, on the filter size and on the number of elements in the filter. This implementation + * uses a variable optimal number of hash functions, depending on the expected + * number of elements. More precisely, a Bloom + * filter for n character sequences with d hash functions will use + * ln 2 dn ≈ 1.44 dn bits; + * false positives will happen with probability 2-d. + * + *

Hash functions are generated at creation time using universal hashing. Each hash function + * uses {@link #NUMBER_OF_WEIGHTS} random integers, which are cyclically multiplied by + * the character codes in a character sequence. The resulting integers are XOR-ed together. + * + *

This class exports access methods that are very similar to those of {@link java.util.Set}, + * but it does not implement that interface, as too many non-optional methods + * would be unimplementable (e.g., iterators). + * + * @author Sebastiano Vigna + */ +public class BloomFilter32bp2 implements Serializable, BloomFilter { + + private static final long serialVersionUID = -2292902803681146635L; + + /** The number of weights used to create hash functions. */ + final public static int NUMBER_OF_WEIGHTS = 2083; // CHANGED FROM 16 + /** The number of bits in this filter. */ + final public long m; + /** the power-of-two that m is */ + final public long power; // 1<n elements, + * false positives will happen with probability 2-d. + */ + public BloomFilter32bp2( final int n, final int d ) { + this.d = d; + long minBits = (long) ((long)n * (long)d / NATURAL_LOG_OF_2); + long pow = 0; + while((1L< 1L<<32 ) { + throw new IllegalArgumentException( "This filter would require " + m + " bits" ); + } + System.out.println("power "+power+" bits "+m+" len "+len); + + bits = new int[ len ]; + + if ( DEBUG ) System.err.println( "Number of bits: " + m ); + + // seeded for reproduceable behavior in repeated runs; BUT: + // SecureRandom's default implementation (as of 1.5) + // seems to mix in its own seeding. + final SecureRandom random = new SecureRandom(new byte[] {19,96}); + weight = new int[ d ][]; + for( int i = 0; i < d; i++ ) { + weight[ i ] = new int[ NUMBER_OF_WEIGHTS ]; + for( int j = 0; j < NUMBER_OF_WEIGHTS; j++ ) + weight[ i ][ j ] = random.nextInt(); + } + } + + /** The number of character sequences in the filter. + * + * @return the number of character sequences in the filter (but see {@link #contains(CharSequence)}). + */ + + public int size() { + return size; + } + + /** Hashes the given sequence with the given hash function. + * + * @param s a character sequence. + * @param l the length of s. + * @param k a hash function index (smaller than {@link #d}). + * @return the position in the filter corresponding to s for the hash function k. + */ + private int hash( final CharSequence s, final int l, final int k ) { + final int[] w = weight[ k ]; + int h = 0, i = l; + while( i-- != 0 ) h ^= s.charAt( i ) * w[ i % NUMBER_OF_WEIGHTS ]; + return h >>> (32-power); + } + + /** Checks whether the given character sequence is in this filter. + * + *

Note that this method may return true on a character sequence that is has + * not been added to the filter. This will happen with probability 2-d, + * where d is the number of hash functions specified at creation time, if + * the number of the elements in the filter is less than n, the number + * of expected elements specified at creation time. + * + * @param s a character sequence. + * @return true if the sequence is in the filter (or if a sequence with the + * same hash sequence is in the filter). + */ + + public boolean contains( final CharSequence s ) { + int i = d, l = s.length(); + while( i-- != 0 ) if ( ! getBit( hash( s, l, i ) ) ) return false; + return true; + } + + /** Adds a character sequence to the filter. + * + * @param s a character sequence. + * @return true if the character sequence was not in the filter (but see {@link #contains(CharSequence)}). + */ + + public boolean add( final CharSequence s ) { + boolean result = false; + int i = d, l = s.length(); + int h; + while( i-- != 0 ) { + h = hash( s, l, i ); + if ( ! getBit( h ) ) result = true; + setBit( h ); + } + if ( result ) size++; + return result; + } + + protected final static int ADDRESS_BITS_PER_UNIT = 5; // 32=2^5 + protected final static int BIT_INDEX_MASK = 31; // = BITS_PER_UNIT - 1; + + /** + * Returns from the local bitvector the value of the bit with + * the specified index. The value is true if the bit + * with the index bitIndex is currently set; otherwise, + * returns false. + * + * (adapted from cern.colt.bitvector.QuickBitVector) + * + * @param bitIndex the bit index. + * @return the value of the bit with the specified index. + */ + protected boolean getBit(int bitIndex) { + return ((bits[(int)(bitIndex >>> ADDRESS_BITS_PER_UNIT)] & (1 << (bitIndex & BIT_INDEX_MASK))) != 0); + } + + /** + * Changes the bit with index bitIndex in local bitvector. + * + * (adapted from cern.colt.bitvector.QuickBitVector) + * + * @param bitIndex the index of the bit to be set. + */ + protected void setBit(int bitIndex) { + bits[(int)(bitIndex >>> ADDRESS_BITS_PER_UNIT)] |= 1 << (bitIndex & BIT_INDEX_MASK); + } + + /* (non-Javadoc) + * @see org.archive.util.BloomFilter#getSizeBytes() + */ + public long getSizeBytes() { + return bits.length*4; + } +} diff --git a/commons/src/main/java/org/archive/util/BloomFilter32bp2Split.java b/commons/src/main/java/org/archive/util/BloomFilter32bp2Split.java new file mode 100644 index 00000000..aba45f75 --- /dev/null +++ b/commons/src/main/java/org/archive/util/BloomFilter32bp2Split.java @@ -0,0 +1,262 @@ +/* BloomFilter +* +* $Id$ +* +* Created on Jun 21, 2005 +* +* Copyright (C) 2005 Internet Archive; a slight adaptation of +* LGPL work (C) Sebastiano Vigna +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +package org.archive.util; + +import java.io.Serializable; +import java.security.SecureRandom; + +/** A Bloom filter. + * + * SLIGHTLY ADAPTED VERSION OF MG4J it.unimi.dsi.mg4j.util.BloomFilter + * + *

KEY CHANGES: + * + *

    + *
  • Adapted to use 32bit ops as much as possible... may be slightly + * faster on 32bit hardware/OS
    • + *
  • Changed to use bitfield that is a power-of-two in size, allowing + * hash() to use bitshifting rather than modulus... may be slightly + * faster
    • + *
  • NUMBER_OF_WEIGHTS is 2083, to better avoid collisions between + * similar strings
    • + *
  • Removed dependence on cern.colt MersenneTwister (replaced with + * SecureRandom) and QuickBitVector (replaced with local methods).
    • + *
+ * + *
+ * + *

Instances of this class represent a set of character sequences (with false positives) + * using a Bloom filter. Because of the way Bloom filters work, + * you cannot remove elements. + * + *

Bloom filters have an expected error rate, depending on the number + * of hash functions used, on the filter size and on the number of elements in the filter. This implementation + * uses a variable optimal number of hash functions, depending on the expected + * number of elements. More precisely, a Bloom + * filter for n character sequences with d hash functions will use + * ln 2 dn ≈ 1.44 dn bits; + * false positives will happen with probability 2-d. + * + *

Hash functions are generated at creation time using universal hashing. Each hash function + * uses {@link #NUMBER_OF_WEIGHTS} random integers, which are cyclically multiplied by + * the character codes in a character sequence. The resulting integers are XOR-ed together. + * + *

This class exports access methods that are very similar to those of {@link java.util.Set}, + * but it does not implement that interface, as too many non-optional methods + * would be unimplementable (e.g., iterators). + * + * @author Sebastiano Vigna + */ +public class BloomFilter32bp2Split implements Serializable, BloomFilter { + + private static final long serialVersionUID = -1504889954381695129L; + + /** The number of weights used to create hash functions. */ + final public static int NUMBER_OF_WEIGHTS = 2083; // CHANGED FROM 16 + /** The number of bits in this filter. */ + final public long m; + /** the power-of-two that m is */ + final public long power; // 1<n elements, + * false positives will happen with probability 2-d. + */ + public BloomFilter32bp2Split( final int n, final int d ) { + this.d = d; + long minBits = (long) ((long)n * (long)d / NATURAL_LOG_OF_2); + long pow = 0; + while((1L< 1L<<32 ) { + throw new IllegalArgumentException( "This filter would require " + m + " bits" ); + } + + aShift = (int) (pow - ADDRESS_BITS_PER_UNIT - 8); + bMask = (1<s. + * @param k a hash function index (smaller than {@link #d}). + * @return the position in the filter corresponding to s for the hash function k. + */ + private int hash( final CharSequence s, final int l, final int k ) { + final int[] w = weight[ k ]; + int h = 0, i = l; + while( i-- != 0 ) h ^= s.charAt( i ) * w[ i % NUMBER_OF_WEIGHTS ]; + return h >>> (32-power); + } + + /** Checks whether the given character sequence is in this filter. + * + *

Note that this method may return true on a character sequence that is has + * not been added to the filter. This will happen with probability 2-d, + * where d is the number of hash functions specified at creation time, if + * the number of the elements in the filter is less than n, the number + * of expected elements specified at creation time. + * + * @param s a character sequence. + * @return true if the sequence is in the filter (or if a sequence with the + * same hash sequence is in the filter). + */ + + public boolean contains( final CharSequence s ) { + int i = d, l = s.length(); + while( i-- != 0 ) if ( ! getBit( hash( s, l, i ) ) ) return false; + return true; + } + + /** Adds a character sequence to the filter. + * + * @param s a character sequence. + * @return true if the character sequence was not in the filter (but see {@link #contains(CharSequence)}). + */ + + public boolean add( final CharSequence s ) { + boolean result = false; + int i = d, l = s.length(); + int h; + while( i-- != 0 ) { + h = hash( s, l, i ); + if ( ! setGetBit( h ) ) result = true; + } + if ( result ) size++; + return result; + } + + protected final static int ADDRESS_BITS_PER_UNIT = 5; // 32=2^5 + protected final static int BIT_INDEX_MASK = 31; // = BITS_PER_UNIT - 1; + + /** + * Returns from the local bitvector the value of the bit with + * the specified index. The value is true if the bit + * with the index bitIndex is currently set; otherwise, + * returns false. + * + * (adapted from cern.colt.bitvector.QuickBitVector) + * + * @param bitIndex the bit index. + * @return the value of the bit with the specified index. + */ + protected boolean getBit(int bitIndex) { + int intIndex = (int)(bitIndex >>> ADDRESS_BITS_PER_UNIT); + return ((bits[intIndex>>>aShift][intIndex&bMask] & (1 << (bitIndex & BIT_INDEX_MASK))) != 0); + } + + /** + * Changes the bit with index bitIndex in local bitvector. + * + * (adapted from cern.colt.bitvector.QuickBitVector) + * + * @param bitIndex the index of the bit to be set. + */ + protected void setBit(int bitIndex) { + int intIndex = (int)(bitIndex >>> ADDRESS_BITS_PER_UNIT); + bits[intIndex>>>aShift][intIndex&bMask] |= 1 << (bitIndex & BIT_INDEX_MASK); + } + + /** + * Sets the bit with index bitIndex in local bitvector -- + * returning the old value. + * + * (adapted from cern.colt.bitvector.QuickBitVector) + * + * @param bitIndex the index of the bit to be set. + */ + protected boolean setGetBit(int bitIndex) { + int intIndex = (int)(bitIndex >>> ADDRESS_BITS_PER_UNIT); + int a = intIndex>>>aShift; + int b = intIndex&bMask; + int mask = 1 << (bitIndex & BIT_INDEX_MASK); + boolean ret = ((bits[a][b] & (mask)) != 0); + bits[a][b] |= mask; + return ret; + } + + /* (non-Javadoc) + * @see org.archive.util.BloomFilter#getSizeBytes() + */ + public long getSizeBytes() { + return bits.length*bits[0].length*4; + } +} diff --git a/commons/src/main/java/org/archive/util/BloomFilter64bit.java b/commons/src/main/java/org/archive/util/BloomFilter64bit.java new file mode 100644 index 00000000..34daad3a --- /dev/null +++ b/commons/src/main/java/org/archive/util/BloomFilter64bit.java @@ -0,0 +1,222 @@ +/* BloomFilter +* +* $Id$ +* +* Created on Jun 21, 2005 +* +* Copyright (C) 2005 Internet Archive; a slight adaptation of +* LGPL work (C) Sebastiano Vigna +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +package org.archive.util; + +import java.io.Serializable; +import java.security.SecureRandom; + +/** A Bloom filter. + * + * SLIGHTLY ADAPTED VERSION OF MG4J it.unimi.dsi.mg4j.util.BloomFilter + * + *

KEY CHANGES: + * + *

    + *
  • NUMBER_OF_WEIGHTS is 2083, to better avoid collisions between + * similar strings
    • + *
  • Removed dependence on cern.colt MersenneTwister (replaced with + * SecureRandom) and QuickBitVector (replaced with local methods).
    • + *
  • Adapted to allow long bit indices so long as the index/64 (used + * an array index in bit vector) fits within Integer.MAX_VALUE
    • + *
+ * + *
+ * + *

Instances of this class represent a set of character sequences (with false positives) + * using a Bloom filter. Because of the way Bloom filters work, + * you cannot remove elements. + * + *

Bloom filters have an expected error rate, depending on the number + * of hash functions used, on the filter size and on the number of elements in the filter. This implementation + * uses a variable optimal number of hash functions, depending on the expected + * number of elements. More precisely, a Bloom + * filter for n character sequences with d hash functions will use + * ln 2 dn ≈ 1.44 dn bits; + * false positives will happen with probability 2-d. + * + *

Hash functions are generated at creation time using universal hashing. Each hash function + * uses {@link #NUMBER_OF_WEIGHTS} random integers, which are cyclically multiplied by + * the character codes in a character sequence. The resulting integers are XOR-ed together. + * + *

This class exports access methods that are very similar to those of {@link java.util.Set}, + * but it does not implement that interface, as too many non-optional methods + * would be unimplementable (e.g., iterators). + * + * @author Sebastiano Vigna + */ +public class BloomFilter64bit implements Serializable, BloomFilter { + + private static final long serialVersionUID = 2317000663009608403L; + + /** The number of weights used to create hash functions. */ + final public static int NUMBER_OF_WEIGHTS = 2083; // CHANGED FROM 16 + /** The number of bits in this filter. */ + final public long m; + /** The number of hash functions used by this filter. */ + final public int d; + /** The underlying bit vector. */ + final private long[] bits; + /** The random integers used to generate the hash functions. */ + final private long[][] weight; + + /** The number of elements currently in the filter. It may be + * smaller than the actual number of additions of distinct character + * sequences because of false positives. + */ + private int size; + + /** The natural logarithm of 2, used in the computation of the number of bits. */ + private final static double NATURAL_LOG_OF_2 = Math.log( 2 ); + + private final static boolean DEBUG = false; + + /** Creates a new Bloom filter with given number of hash functions and expected number of elements. + * + * @param n the expected number of elements. + * @param d the number of hash functions; if the filter add not more than n elements, + * false positives will happen with probability 2-d. + */ + public BloomFilter64bit( final int n, final int d ) { + this.d = d; + int len = (int)Math.ceil( ( (long)n * (long)d / NATURAL_LOG_OF_2 ) / 64L ); + if ( len/64 > Integer.MAX_VALUE ) throw new IllegalArgumentException( "This filter would require " + len * 64L + " bits" ); + bits = new long[ len ]; + m = bits.length * 64; + + if ( DEBUG ) System.err.println( "Number of bits: " + m ); + + // seeded for reproduceable behavior in repeated runs; BUT: + // SecureRandom's default implementation (as of 1.5) + // seems to mix in its own seeding. + final SecureRandom random = new SecureRandom(new byte[] {19,96}); + weight = new long[ d ][]; + for( int i = 0; i < d; i++ ) { + weight[ i ] = new long[ NUMBER_OF_WEIGHTS ]; + for( int j = 0; j < NUMBER_OF_WEIGHTS; j++ ) + weight[ i ][ j ] = random.nextLong(); + } + } + + /** The number of character sequences in the filter. + * + * @return the number of character sequences in the filter (but see {@link #contains(CharSequence)}). + */ + + public int size() { + return size; + } + + /** Hashes the given sequence with the given hash function. + * + * @param s a character sequence. + * @param l the length of s. + * @param k a hash function index (smaller than {@link #d}). + * @return the position in the filter corresponding to s for the hash function k. + */ + + private long hash( final CharSequence s, final int l, final int k ) { + final long[] w = weight[ k ]; + long h = 0; + int i = l; + while( i-- != 0 ) h ^= s.charAt( i ) * w[ i % NUMBER_OF_WEIGHTS ]; + return ( h & 0x7FFFFFFFFFFFFFFFL ) % m; + } + + /** Checks whether the given character sequence is in this filter. + * + *

Note that this method may return true on a character sequence that is has + * not been added to the filter. This will happen with probability 2-d, + * where d is the number of hash functions specified at creation time, if + * the number of the elements in the filter is less than n, the number + * of expected elements specified at creation time. + * + * @param s a character sequence. + * @return true if the sequence is in the filter (or if a sequence with the + * same hash sequence is in the filter). + */ + + public boolean contains( final CharSequence s ) { + int i = d, l = s.length(); + while( i-- != 0 ) if ( ! getBit( hash( s, l, i ) ) ) return false; + return true; + } + + /** Adds a character sequence to the filter. + * + * @param s a character sequence. + * @return true if the character sequence was not in the filter (but see {@link #contains(CharSequence)}). + */ + + public boolean add( final CharSequence s ) { + boolean result = false; + int i = d, l = s.length(); + long h; + while( i-- != 0 ) { + h = hash( s, l, i ); + if ( ! getBit( h ) ) result = true; + setBit( h ); + } + if ( result ) size++; + return result; + } + + protected final static long ADDRESS_BITS_PER_UNIT = 6; // 64=2^6 + protected final static long BIT_INDEX_MASK = 63; // = BITS_PER_UNIT - 1; + + /** + * Returns from the local bitvector the value of the bit with + * the specified index. The value is true if the bit + * with the index bitIndex is currently set; otherwise, + * returns false. + * + * (adapted from cern.colt.bitvector.QuickBitVector) + * + * @param bitIndex the bit index. + * @return the value of the bit with the specified index. + */ + protected boolean getBit(long bitIndex) { + return ((bits[(int)(bitIndex >> ADDRESS_BITS_PER_UNIT)] & (1L << (bitIndex & BIT_INDEX_MASK))) != 0); + } + + /** + * Changes the bit with index bitIndex in local bitvector. + * + * (adapted from cern.colt.bitvector.QuickBitVector) + * + * @param bitIndex the index of the bit to be set. + */ + protected void setBit( long bitIndex) { + bits[(int)(bitIndex >> ADDRESS_BITS_PER_UNIT)] |= 1L << (bitIndex & BIT_INDEX_MASK); + } + + /* (non-Javadoc) + * @see org.archive.util.BloomFilter#getSizeBytes() + */ + public long getSizeBytes() { + return bits.length*8; + } +} diff --git a/commons/src/main/java/org/archive/util/CachedBdbMap.java b/commons/src/main/java/org/archive/util/CachedBdbMap.java new file mode 100644 index 00000000..d0278b60 --- /dev/null +++ b/commons/src/main/java/org/archive/util/CachedBdbMap.java @@ -0,0 +1,667 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.util; + +import java.io.File; +import java.io.IOException; +import java.io.Serializable; +import java.lang.ref.PhantomReference; +import java.lang.ref.Reference; +import java.lang.ref.ReferenceQueue; +import java.lang.ref.SoftReference; +import java.lang.reflect.Field; +import java.util.AbstractMap; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.Map; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; + +import com.sleepycat.bind.EntryBinding; +import com.sleepycat.bind.serial.SerialBinding; +import com.sleepycat.bind.serial.StoredClassCatalog; +import com.sleepycat.bind.tuple.TupleBinding; +import com.sleepycat.collections.StoredSortedMap; +import com.sleepycat.je.Database; +import com.sleepycat.je.DatabaseConfig; +import com.sleepycat.je.DatabaseException; +import com.sleepycat.je.Environment; +import com.sleepycat.je.EnvironmentConfig; + +/** + * A BDB JE backed hashmap. It extends the normal BDB JE map implementation by + * holding a cache of soft referenced objects. That is objects are not written + * to disk until they are not referenced by any other object and therefore can be + * Garbage Collected. + * + * @author John Erik Halse + * @author stack + * @author gojomo + * + */ +@SuppressWarnings("unchecked") +public class CachedBdbMap extends AbstractMap +implements Map, Serializable { + + private static final long serialVersionUID = -8655539411367047332L; + + private static final Logger logger = + Logger.getLogger(CachedBdbMap.class.getName()); + + /** The database name of the class definition catalog.*/ + private static final String CLASS_CATALOG = "java_class_catalog"; + + /** + * A map of BDB JE Environments so that we reuse the Environment for + * databases in the same directory. + */ + private static final Map dbEnvironmentMap = + new HashMap(); + + /** The BDB JE environment used for this instance. + */ + private transient DbEnvironmentEntry dbEnvironment; + + /** The BDB JE database used for this instance. */ + protected transient Database db; + + /** The Collection view of the BDB JE database used for this instance. */ + protected transient StoredSortedMap diskMap; + + /** The softreferenced cache */ + private transient Map> memMap; + + protected transient ReferenceQueue refQueue; + + /** The number of objects in the diskMap StoredMap. + * (Package access for unit testing.) */ + protected int diskMapSize = 0; + + /** + * Count of times we got an object from in-memory cache. + */ + private long cacheHit = 0; + + /** + * Count of times the {@link CachedBdbMap#get(Object)} method was called. + */ + private long countOfGets = 0; + + /** + * Count of every time we went to the disk-based map AND we found an + * object (Doesn't include accesses that came back null). + */ + private long diskHit = 0; + + /** + * Name of bdbje db. + */ + private String dbName = null; + + + private Class keyClass; + + + private Class valueClass; + + + /** + * Reference to the Reference#referent Field. + */ + protected static Field referentField; + static { + // We need access to the referent field in the PhantomReference. + // For more on this trick, see + // http://www.javaspecialists.co.za/archive/Issue098.html and for + // discussion: + // http://www.theserverside.com/tss?service=direct/0/NewsThread/threadViewer.markNoisy.link&sp=l29865&sp=l146901 + try { + referentField = Reference.class.getDeclaredField("referent"); + referentField.setAccessible(true); + } catch (SecurityException e) { + throw new RuntimeException(e); + } catch (NoSuchFieldException e) { + throw new RuntimeException(e); + } + } + + /** + * Simple structure to keep needed information about a DB Environment. + */ + protected static class DbEnvironmentEntry { + Environment environment; + StoredClassCatalog classCatalog; + int openDbCount = 0; + File dbDir; + } + + /** + * Shudown default constructor. + */ + private CachedBdbMap() { + super(); + } + + /** + * Constructor. + * + * You must call + * {@link #initialize(Environment, Class, Class, StoredClassCatalog)} + * to finish construction. Construction is two-stepped to support + * reconnecting a deserialized CachedBdbMap with its backing bdbje + * database. + * + * @param dbName Name of the backing db this instance should use. + */ + public CachedBdbMap(final String dbName) { + this(); + this.dbName = dbName; + } + + /** + * A constructor for creating a new CachedBdbMap. + * + * Even though the put and get methods conforms to the Collections interface + * taking any object as key or value, you have to submit the class of the + * allowed key and value objects here and will get an exception if you try + * to put anything else in the map. + * + *

This constructor internally calls + * {@link #initialize(Environment, Class, Class, StoredClassCatalog)}. + * Do not call initialize if you use this constructor. + * + * @param dbDir The directory where the database will be created. + * @param dbName The name of the database to back this map by. + * @param keyClass The class of the objects allowed as keys. + * @param valueClass The class of the objects allowed as values. + * + * @throws DatabaseException is thrown if the underlying BDB JE database + * throws an exception. + */ + public CachedBdbMap(final File dbDir, final String dbName, + final Class keyClass, final Class valueClass) + throws DatabaseException { + this(dbName); + this.dbEnvironment = getDbEnvironment(dbDir); + this.dbEnvironment.openDbCount++; + initialize(dbEnvironment.environment, keyClass, valueClass, + dbEnvironment.classCatalog); + if (logger.isLoggable(Level.INFO)) { + // Write out the bdb configuration. + EnvironmentConfig cfg = this.dbEnvironment.environment.getConfig(); + logger.info("BdbConfiguration: Cache percentage " + + cfg.getCachePercent() + ", cache size " + cfg.getCacheSize() + + ", Map size: " + size()); + } + } + + /** + * Call this method when you have an instance when you used the + * default constructor or when you have a deserialized instance that you + * want to reconnect with an extant bdbje environment. Do not + * call this method if you used the + * {@link #CachedBdbMap(File, String, Class, Class)} constructor. + * @param env + * @param keyClass + * @param valueClass + * @param classCatalog + * @throws DatabaseException + */ + @SuppressWarnings("unchecked") + public synchronized void initialize(final Environment env, final Class keyClass, + final Class valueClass, final StoredClassCatalog classCatalog) + throws DatabaseException { + initializeInstance(); + this.db = openDatabase(env, this.dbName); + this.keyClass = keyClass; + this.valueClass = valueClass; + this.diskMap = createDiskMap(this.db, classCatalog, keyClass, + valueClass); + } + + + public Class getKeyClass() { + return keyClass; + } + + + public Class getValueClass() { + return valueClass; + } + + /** + * Do any instance setup. + * This method is used by constructors and when deserializing an instance. + */ + protected void initializeInstance() { + this.memMap = new HashMap>(); + this.refQueue = new ReferenceQueue(); + } + + protected StoredSortedMap createDiskMap(Database database, + StoredClassCatalog classCatalog, Class keyClass, Class valueClass) { + EntryBinding keyBinding = TupleBinding.getPrimitiveBinding(keyClass); + if(keyBinding == null) { + keyBinding = new SerialBinding(classCatalog, keyClass); + } + EntryBinding valueBinding = TupleBinding.getPrimitiveBinding(valueClass); + if(valueBinding == null) { + valueBinding = new SerialBinding(classCatalog, valueClass); + } + return new StoredSortedMap(database, keyBinding, valueBinding, true); + } + + /** + * Get the database environment for a physical directory where data will be + * stored. + *

+ * If the environment already exist it will be reused, else a new one will + * be created. + * + * @param dbDir The directory where BDB JE data will be stored. + * @return a datastructure containing the environment and a default database + * for storing class definitions. + */ + private DbEnvironmentEntry getDbEnvironment(File dbDir) { + if (dbEnvironmentMap.containsKey(dbDir.getAbsolutePath())) { + return (DbEnvironmentEntry) dbEnvironmentMap.get(dbDir + .getAbsolutePath()); + } + EnvironmentConfig envConfig = new EnvironmentConfig(); + envConfig.setAllowCreate(true); + envConfig.setTransactional(false); + + // We're doing the caching ourselves so setting these at the lowest + // possible level. + envConfig.setCachePercent(1); + DbEnvironmentEntry env = new DbEnvironmentEntry(); + try { + env.environment = new Environment(dbDir, envConfig); + env.dbDir = dbDir; + dbEnvironmentMap.put(dbDir.getAbsolutePath(), env); + + DatabaseConfig dbConfig = new DatabaseConfig(); + dbConfig.setTransactional(false); + dbConfig.setAllowCreate(true); + dbConfig.setDeferredWrite(true); + + Database catalogDb = env.environment.openDatabase(null, + CLASS_CATALOG, dbConfig); + + env.classCatalog = new StoredClassCatalog(catalogDb); + } catch (DatabaseException e) { + e.printStackTrace(); + //throw new FatalConfigurationException(e.getMessage()); + } + return env; + } + + protected Database openDatabase(final Environment environment, + final String dbName) throws DatabaseException { + DatabaseConfig dbConfig = new DatabaseConfig(); + dbConfig.setTransactional(false); + dbConfig.setAllowCreate(true); + dbConfig.setDeferredWrite(true); + return environment.openDatabase(null, dbName, dbConfig); + } + + public synchronized void close() throws DatabaseException { + // Close out my bdb db. + if (this.db != null) { + try { + this.db.sync(); + this.db.close(); + } catch (DatabaseException e) { + e.printStackTrace(); + } finally { + this.db = null; + } + } + if (dbEnvironment != null) { + dbEnvironment.openDbCount--; + if (dbEnvironment.openDbCount <= 0) { + dbEnvironment.classCatalog.close(); + dbEnvironment.environment.close(); + dbEnvironmentMap.remove(dbEnvironment.dbDir.getAbsolutePath()); + dbEnvironment = null; + } + } + } + + protected void finalize() throws Throwable { + close(); + super.finalize(); + } + + /** + * The keySet of the diskMap is all relevant keys. + * + * @see java.util.Map#keySet() + */ + @SuppressWarnings("unchecked") + public Set keySet() { + return diskMap.keySet(); + } + + public Set> entrySet() { + // Would require complicated implementation to + // maintain identity guarantees, so skipping + throw new UnsupportedOperationException(); + } + + public synchronized V get(final Object object) { + K key = toKey(object); + countOfGets++; + expungeStaleEntries(); + if (countOfGets % 10000 == 0) { + logCacheSummary(); + } + SoftEntry entry = memMap.get(key); + if (entry != null) { + V val = entry.get(); // get & hold, so not cleared pre-return + if (val != null) { + cacheHit++; + return val; + } + // Explicitly clear this entry from referencequeue since its + // value is null. + expungeStaleEntry(entry); + } + + // check backing diskMap + V v = diskMapGet(key); + if (v != null) { + diskHit++; + memMap.put(key, new SoftEntry(key, v, refQueue)); + } + return v; + } + + /** + * Info to log, if at FINE level, on every get() + */ + private void logCacheSummary() { + if (!logger.isLoggable((Level.FINE))) { + return; + } + if (cacheHit + diskHit == 0) { + // Preent division by zero below + return; + } + try { + long cacheHitPercent = (cacheHit * 100) / (cacheHit + diskHit); + logger.fine("DB name: " + this.db.getDatabaseName() + + ", Cache Hit: " + cacheHitPercent + + "%, Not in map: " + (countOfGets - (cacheHit + diskHit)) + + ", Total number of gets: " + countOfGets); + } catch (DatabaseException e) { + // This is just for logging so ignore DB Exceptions + } + } + + public synchronized V put(K key, V value) { + V prevVal = get(key); + memMap.put(key, new SoftEntry(key, value, refQueue)); + diskMap.put(key,value); // dummy + if(prevVal==null) { + diskMapSize++; + } + return prevVal; + } + + /** + * Note that a call to this method CLOSEs the underlying bdbje. + * This instance is no longer of any use. It must be re-initialized. + * We close the db here because if this BigMap is being treated as a plain + * Map, this is only opportunity for cleanup. + */ + public synchronized void clear() { + this.memMap.clear(); + if (db != null) { + this.diskMap.clear(); + this.diskMapSize = 0; + try { + close(); + } catch (DatabaseException e) { + e.printStackTrace(); + } + } + } + + public synchronized V remove(final Object key) { + V prevValue = get(key); + memMap.remove(key); + expungeStaleEntries(); + diskMap.remove(key); + diskMapSize--; + return prevValue; + } + + public synchronized boolean containsKey(Object key) { + if (quickContainsKey(key)) { + return true; + } + return diskMap.containsKey(key); + } + + public synchronized boolean quickContainsKey(Object key) { + expungeStaleEntries(); + return memMap.containsKey(key); + } + + public synchronized boolean containsValue(Object value) { + if (quickContainsValue(value)) { + return true; + } + return diskMap.containsValue(value); + } + + public synchronized boolean quickContainsValue(Object value) { + expungeStaleEntries(); + // FIXME this isn't really right, as memMap is of SoftEntries + return memMap.containsValue(value); + } + + public int size() { + return diskMapSize; + } + + protected String getDatabaseName() { + String name = "DbName-Lookup-Failed"; + try { + if (this.db != null) { + name = this.db.getDatabaseName(); + } + } catch (DatabaseException e) { + // Ignore. + } + return name; + } + + /** + * Sync in-memory map entries to backing disk store. + * When done, the memory map will be cleared and all entries stored + * on disk. + */ + public synchronized void sync() { + String dbName = null; + // Sync. memory and disk. + long startTime = 0; + if (logger.isLoggable(Level.INFO)) { + dbName = getDatabaseName(); + startTime = System.currentTimeMillis(); + logger.info(dbName + " start sizes: disk " + this.diskMapSize + + ", mem " + this.memMap.size()); + } + expungeStaleEntries(); + LinkedList stale = new LinkedList(); + for (Iterator i = this.memMap.keySet().iterator(); i.hasNext();) { + Object key = i.next(); + SoftEntry entry = (SoftEntry) memMap.get(key); + if (entry != null) { + // Get & hold so not cleared pre-return. + Object value = entry.get(); + if (value != null) { + this.diskMap.put(key, value); + } else { + stale.add(entry); + } + } + } + // for any entries above that had been cleared, ensure expunged + for (SoftEntry entry : stale) { + expungeStaleEntry(entry); + } + + // force sync of deferred-writes + try { + this.db.sync(); + } catch (DatabaseException e) { + // TODO Auto-generated catch block + throw new RuntimeException(e); + } + + if (logger.isLoggable(Level.INFO)) { + logger.info(dbName + " sync took " + + (System.currentTimeMillis() - startTime) + "ms. " + + "Finish sizes: disk " + + this.diskMapSize + ", mem " + this.memMap.size()); + } + } + + private void expungeStaleEntries() { + int c = 0; + for(SoftEntry entry; (entry = refQueuePoll()) != null;) { + expungeStaleEntry(entry); + c++; + } + if (c > 0 && logger.isLoggable(Level.FINER)) { + try { + logger.finer("DB: " + db.getDatabaseName() + ", Expunged: " + + c + ", Diskmap size: " + diskMapSize + + ", Cache size: " + memMap.size()); + } catch (DatabaseException e) { + // Just for logging so ignore Exceptions + } + } + } + + private void expungeStaleEntry(SoftEntry entry) { + // If phantom already null, its already expunged -- probably + // because it was purged directly first from inside in + // {@link #get(String)} and then it went on the poll queue and + // when it came off inside in expungeStaleEntries, this method + // was called again. + if (entry.getPhantom() == null) { + return; + } + // If the object that is in memMap is not the one passed here, then + // memMap has been changed -- probably by a put on top of this entry. + if (memMap.get(entry.getPhantom().getKey()) == entry) { + memMap.remove(entry.getPhantom().getKey()); + diskMap.put(entry.getPhantom().getKey(), + entry.getPhantom().doctoredGet()); + } + entry.clearPhantom(); + } + + private class PhantomEntry extends PhantomReference { + private final Object key; + + public PhantomEntry(Object key, T referent) { + super(referent, null); + this.key = key; + } + + /** + * @return Return the referent. The contract for {@link #get()} + * always returns a null referent. We've cheated and doctored + * PhantomReference to return the actual referent value. See notes + * at {@link #referentField}; + */ + public Object doctoredGet() { + try { + // Here we use the referentField saved off on static + // initialization of this class to get at this References' + // private referent field. + return referentField.get(this); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + /** + * @return Returns the key. + */ + public Object getKey() { + return this.key; + } + } + + private class SoftEntry extends SoftReference { + private PhantomEntry phantom; + + public SoftEntry(Object key, T referent, ReferenceQueue q) { + super(referent, q); + this.phantom = new PhantomEntry(key, referent); + } + + /** + * @return Returns the phantom reference. + */ + public PhantomEntry getPhantom() { + return this.phantom; + } + + public void clearPhantom() { + this.phantom.clear(); + this.phantom = null; + super.clear(); + } + } + + private void readObject(java.io.ObjectInputStream stream) + throws IOException, ClassNotFoundException { + stream.defaultReadObject(); + initializeInstance(); + if (logger.isLoggable(Level.FINE)) { + logger.fine(getDatabaseName() + " diskMapSize: " + diskMapSize); + } + } + + + + @SuppressWarnings("unchecked") + private K toKey(Object o) { + return (K)o; + } + + @SuppressWarnings("unchecked") + private V diskMapGet(K k) { + return (V)diskMap.get(k); + } + + @SuppressWarnings("unchecked") + private SoftEntry refQueuePoll() { + return (SoftEntry)refQueue.poll(); + } +} diff --git a/commons/src/main/java/org/archive/util/CollectionUtils.java b/commons/src/main/java/org/archive/util/CollectionUtils.java new file mode 100644 index 00000000..8f17b53d --- /dev/null +++ b/commons/src/main/java/org/archive/util/CollectionUtils.java @@ -0,0 +1,48 @@ +/* Copyright (C) 2007 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * CollectionUtils.java + * Created on January 23, 2007 + * + * $Header:$ + */ +package org.archive.util; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class CollectionUtils { + + + + private CollectionUtils() { + + } + + + + public static Set constantSet(T... objects) { + List list = Arrays.asList(objects); + HashSet set = new HashSet(list); + return Collections.unmodifiableSet(set); + } + +} diff --git a/commons/src/main/java/org/archive/util/DNSJavaUtil.java b/commons/src/main/java/org/archive/util/DNSJavaUtil.java new file mode 100644 index 00000000..470f93b6 --- /dev/null +++ b/commons/src/main/java/org/archive/util/DNSJavaUtil.java @@ -0,0 +1,80 @@ +/* DNSJavaUtil + * + * Created on Oct 8, 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.util; + +import java.net.InetAddress; + +import org.xbill.DNS.ARecord; +import org.xbill.DNS.DClass; +import org.xbill.DNS.Record; +import org.xbill.DNS.TextParseException; +import org.xbill.DNS.Type; +import org.xbill.DNS.Lookup;; + +/** + * Utility methods based on DNSJava. + * Use these utilities to avoid having to use the native InetAddress lookup. + * @author stack + * @version $Date$, $Revision$ + */ +public class DNSJavaUtil { + private DNSJavaUtil() { + super(); + } + + /** + * Return an InetAddress for passed host. + * + * If passed host is an IPv4 address, we'll not do a DNSJava + * lookup. + * + * @param host Host to lookup in dnsjava. + * @return A host address or null if not found. + */ + public static InetAddress getHostAddress(String host) { + InetAddress hostAddress = InetAddressUtil.getIPHostAddress(host); + if (hostAddress != null) { + return hostAddress; + } + + // Ask dnsjava for the inetaddress. Should be in its cache. + Record[] rrecordSet; + try { + rrecordSet = (new Lookup(host, Type.A, DClass.IN)).run(); + } catch (TextParseException e) { + rrecordSet = null; + } + if (rrecordSet != null) { + // Get TTL and IP info from the first A record (there may be + // multiple, e.g. www.washington.edu). + for (int i = 0; i < rrecordSet.length; i++) { + if (rrecordSet[i].getType() != Type.A) { + continue; + } + hostAddress = ((ARecord)rrecordSet[i]).getAddress(); + break; + } + } + return hostAddress; + } +} diff --git a/commons/src/main/java/org/archive/util/DevUtils.java b/commons/src/main/java/org/archive/util/DevUtils.java new file mode 100644 index 00000000..a0f7e24a --- /dev/null +++ b/commons/src/main/java/org/archive/util/DevUtils.java @@ -0,0 +1,120 @@ +/* DevUtils + * + * Created on Oct 29, 2003 + * + * Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.util; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.logging.Logger; + + +/** + * Write a message and stack trace to the 'org.archive.util.DevUtils' logger. + * + * @author gojomo + * @version $Revision$ $Date$ + */ +public class DevUtils { + public static Logger logger = + Logger.getLogger(DevUtils.class.getName()); + + /** + * Log a warning message to the logger 'org.archive.util.DevUtils' made of + * the passed 'note' and a stack trace based off passed exception. + * + * @param ex Exception we print a stacktrace on. + * @param note Message to print ahead of the stacktrace. + */ + public static void warnHandle(Throwable ex, String note) { + logger.warning(TextUtils.exceptionToString(note, ex)); + } + + /** + * @return Extra information gotten from current Thread. May not + * always be available in which case we return empty string. + */ + public static String extraInfo() { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + final Thread current = Thread.currentThread(); + if (current instanceof MultiReporter) { + MultiReporter tt = (MultiReporter)current; + try { + tt.reportTo(pw); + } catch (IOException e) { + // Not really possible w/ a StringWriter + e.printStackTrace(); + } + } + if (current instanceof ProgressStatisticsReporter) { + ProgressStatisticsReporter tt = (ProgressStatisticsReporter)current; + try { + tt.progressStatisticsLegend(pw); + tt.progressStatisticsLine(pw); + } catch (IOException e) { + // Not really possible w/ a StringWriter + e.printStackTrace(); + } + } + pw.flush(); + return sw.toString(); + } + + /** + * Nothing to see here, move along. + * @deprecated This method was never used. + */ + @Deprecated + public static void betterPrintStack(RuntimeException re) { + re.printStackTrace(System.err); + } + + /** + * Send this JVM process a SIGQUIT; giving a thread dump and possibly + * a heap histogram (if using -XX:+PrintClassHistogram). + * + * Used to automatically dump info, for example when a serious error + * is encountered. Would use 'jmap'/'jstack', but have seen JVM + * lockups -- perhaps due to lost thread wake signals -- when using + * those against Sun 1.5.0+03 64bit JVM. + */ + public static void sigquitSelf() { + try { + Process p = Runtime.getRuntime().exec( + new String[] {"perl", "-e", "print getppid(). \"\n\";"}); + BufferedReader br = + new BufferedReader(new InputStreamReader(p.getInputStream())); + String ppid = br.readLine(); + Runtime.getRuntime().exec( + new String[] {"sh", "-c", "kill -3 "+ppid}).waitFor(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } +} diff --git a/commons/src/main/java/org/archive/util/FileUtils.java b/commons/src/main/java/org/archive/util/FileUtils.java new file mode 100644 index 00000000..948edd7c --- /dev/null +++ b/commons/src/main/java/org/archive/util/FileUtils.java @@ -0,0 +1,612 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.util; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileFilter; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.channels.ClosedByInterruptException; +import java.nio.channels.FileChannel; +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Pattern; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.filefilter.IOFileFilter; +import org.apache.commons.lang.math.LongRange; + + +/** Utility methods for manipulating files and directories. + * + * @contributor John Erik Halse + * @contributor gojomo + */ +public class FileUtils { + private static final Logger LOGGER = + Logger.getLogger(FileUtils.class.getName()); + + /** + * Constructor made private because all methods of this class are static. + */ + private FileUtils() { + super(); + } + + /** Recursively copy all files from one directory to another. + * + * @param src file or directory to copy from. + * @param dest file or directory to copy to. + * @throws IOException + * @deprecated use org.apache.commons.io.FileUtils.copyDirectory() + */ + public static void copyFiles(File src, File dest) + throws IOException { + org.apache.commons.io.FileUtils.copyDirectory(src, dest); + } + + /** + * Copy the src file to the destination. Deletes any preexisting + * file at destination. + * + * @param src + * @param dest + * @return True if the extent was greater than actual bytes copied. + * @throws FileNotFoundException + * @throws IOException + */ + public static boolean copyFile(final File src, final File dest) + throws FileNotFoundException, IOException { + return copyFile(src, dest, -1, true); + } + + /** + * Copy the src file to the destination. + * + * @param src + * @param dest + * @param overwrite If target file already exits, and this parameter is + * true, overwrite target file (We do this by first deleting the target + * file before we begin the copy). + * @return True if the extent was greater than actual bytes copied. + * @throws FileNotFoundException + * @throws IOException + * @deprecated use org.apache.commons.io.FileUtils.co + */ + public static boolean copyFile(final File srcFile, final File destFile, + final boolean overwrite) + throws FileNotFoundException, IOException { + org.apache.commons.io.FileUtils.copyFile(srcFile, destFile); + return false; + } + + /** + * Copy up to extent bytes of the source file to the destination. + * Deletes any preexisting file at destination. + * + * @param src + * @param dest + * @param extent Maximum number of bytes to copy + * @return True if the extent was greater than actual bytes copied. + * @throws FileNotFoundException + * @throws IOException + */ + public static boolean copyFile(final File src, final File dest, + long extent) + throws FileNotFoundException, IOException { + return copyFile(src, dest, extent, true); + } + + /** + * Copy up to extent bytes of the source file to the destination + * + * @param src + * @param dest + * @param extent Maximum number of bytes to copy + * @param overwrite If target file already exits, and this parameter is + * true, overwrite target file (We do this by first deleting the target + * file before we begin the copy). + * @return True if the extent was greater than actual bytes copied. + * @throws FileNotFoundException + * @throws IOException + */ + public static boolean copyFile(final File src, final File dest, + long extent, final boolean overwrite) + throws FileNotFoundException, IOException { + boolean result = false; + if (LOGGER.isLoggable(Level.FINE)) { + LOGGER.fine("Copying file " + src + " to " + dest + " extent " + + extent + " exists " + dest.exists()); + } + if (dest.exists()) { + if (overwrite) { + dest.delete(); + LOGGER.finer(dest.getAbsolutePath() + " removed before copy."); + } else { + // Already in place and we're not to overwrite. Return. + return result; + } + } + FileInputStream fis = null; + FileOutputStream fos = null; + FileChannel fcin = null; + FileChannel fcout = null; + try { + // Get channels + fis = new FileInputStream(src); + fos = new FileOutputStream(dest); + fcin = fis.getChannel(); + fcout = fos.getChannel(); + if (extent < 0) { + extent = fcin.size(); + } + + // Do the file copy + long trans = fcin.transferTo(0, extent, fcout); + if (trans < extent) { + result = false; + } + result = true; + } catch (IOException e) { + // Add more info to the exception. Preserve old stacktrace. + // We get 'Invalid argument' on some file copies. See + // http://intellij.net/forums/thread.jsp?forum=13&thread=63027&message=853123 + // for related issue. + String message = "Copying " + src.getAbsolutePath() + " to " + + dest.getAbsolutePath() + " with extent " + extent + + " got IOE: " + e.getMessage(); + if ((e instanceof ClosedByInterruptException) || + ((e.getMessage()!=null) + &&e.getMessage().equals("Invalid argument"))) { + LOGGER.severe("Failed copy, trying workaround: " + message); + workaroundCopyFile(src, dest); + } else { + IOException newE = new IOException(message); + newE.initCause(e); + throw newE; + } + } finally { + // finish up + if (fcin != null) { + fcin.close(); + } + if (fcout != null) { + fcout.close(); + } + if (fis != null) { + fis.close(); + } + if (fos != null) { + fos.close(); + } + } + return result; + } + + protected static void workaroundCopyFile(final File src, + final File dest) + throws IOException { + FileInputStream from = null; + FileOutputStream to = null; + try { + from = new FileInputStream(src); + to = new FileOutputStream(dest); + byte[] buffer = new byte[4096]; + int bytesRead; + while ((bytesRead = from.read(buffer)) != -1) { + to.write(buffer, 0, bytesRead); + } + } finally { + if (from != null) { + try { + from.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + if (to != null) { + try { + to.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + } + + /** Deletes all files and subdirectories under dir. + * @param dir + * @return true if all deletions were successful. If a deletion fails, the + * method stops attempting to delete and returns false. + * @deprecated use org.apache.commons.io.FileUtils.deleteDirectory() + */ + public static boolean deleteDir(File dir) { + if (dir.isDirectory()) { + String[] children = dir.list(); + for (int i=0; i to insert found lines + * @param lineEstimate int estimate of line size, 0 means use default + * of 128 + * @return LongRange indicating the file offsets corresponding to + * the beginning of the first line returned, and the point + * after the end of the last line returned + * @throws IOException + */ + @SuppressWarnings("unchecked") + public static LongRange pagedLines(File file, long position, + int signedDesiredLineCount, List lines, int lineEstimate) + throws IOException { + // consider negative positions as from end of file; -1 = last byte + if (position < 0) { + position = file.length() + position; + } + + // calculate a reasonably sized chunk likely to have all desired lines + if(lineEstimate == 0) { + lineEstimate = 128; + } + int desiredLineCount = Math.abs(signedDesiredLineCount); + long startPosition; + long fileEnd = file.length(); + int bufferSize = (desiredLineCount + 5) * lineEstimate; + if(signedDesiredLineCount>0) { + // reading forward; include previous char in case line-end + startPosition = position - 1; + } else { + // reading backward + startPosition = position - bufferSize + (2 * lineEstimate); + } + if(startPosition<0) { + startPosition = 0; + } + if(startPosition+bufferSize > fileEnd) { + bufferSize = (int)(fileEnd - startPosition); + } + + // read that reasonable chunk + FileInputStream fis = new FileInputStream(file); + fis.getChannel().position(startPosition); + byte[] buf = new byte[bufferSize]; + IoUtils.readFully(fis, buf); + IOUtils.closeQuietly(fis); + + // find all line starts fully in buffer + // (positions after a line-end, per line-end definition in + // BufferedReader.readLine) + LinkedList lineStarts = new LinkedList(); + if(startPosition==0) { + lineStarts.add(0); + } + boolean atLineEnd = false; + boolean eatLF = false; + int i; + for(i = 0; i < bufferSize; i++) { + if ((char) buf[i] == '\n' && eatLF) { + eatLF = false; + continue; + } + if(atLineEnd) { + atLineEnd = false; + lineStarts.add(i); + if(signedDesiredLineCount<0 && startPosition+i > position) { + // reached next line past position, read no more + break; + } + } + if ((char) buf[i] == '\r') { + atLineEnd = true; + eatLF = true; + continue; + } + if ((char) buf[i] == '\n') { + atLineEnd = true; + } + } + if(startPosition+i == fileEnd) { + // add phantom lineStart after end + lineStarts.add(bufferSize); + } + int foundFullLines = lineStarts.size()-1; + + // if found no lines + if(foundFullLines<1) { + if(signedDesiredLineCount>0) { + if(startPosition+bufferSize == fileEnd) { + // nothing more to read: return nothing + return new LongRange(fileEnd,fileEnd); + } else { + // retry with larger lineEstimate + return pagedLines(file, position, signedDesiredLineCount, lines, Math.max(bufferSize,lineEstimate)); + } + + } else { + // try again with much larger line estimate + // TODO: fail gracefully before growing to multi-MB buffers + return pagedLines(file, position, signedDesiredLineCount, lines, bufferSize); + } + } + + // trim unneeded lines + while(signedDesiredLineCount>0 && startPosition+lineStarts.getFirst()desiredLineCount+1) { + if (signedDesiredLineCount < 0 && (startPosition+lineStarts.get(1) <= position) ) { + // discard from front until reach line containing target position + lineStarts.removeFirst(); + } else { + lineStarts.removeLast(); + } + } + int firstLine = lineStarts.getFirst(); + int partialLine = lineStarts.getLast(); + LongRange range = new LongRange(startPosition + firstLine, startPosition + partialLine); + List foundLines = + IOUtils.readLines(new ByteArrayInputStream(buf,firstLine,partialLine-firstLine)); + + if(foundFullLines 0) { + // if needed and reading backward, read more lines from earlier + range = expandRange( + range, + pagedLines(file, + range.getMinimumLong()-1, + signedDesiredLineCount+foundFullLines, + lines, + bufferSize/foundFullLines)); + + } + + lines.addAll(foundLines); + + if(signedDesiredLineCount < 0 && range.getMaximumLong() < position) { + // did not get line containining start position + range = expandRange( + range, + pagedLines(file, + partialLine, + 1, + lines, + bufferSize/foundFullLines)); + } + + if(signedDesiredLineCount > 0 && foundFullLines < desiredLineCount && range.getMaximumLong() < fileEnd) { + // need more forward lines + range = expandRange( + range, + pagedLines(file, + range.getMaximumLong(), + desiredLineCount - foundFullLines, + lines, + bufferSize/foundFullLines)); + } + + return range; + } + + public static LongRange expandRange(LongRange range1, LongRange range2) { + return new LongRange(Math.min(range1.getMinimumLong(), range2.getMinimumLong()), + Math.max(range1.getMaximumLong(), range2.getMaximumLong())); + + } + + public static LongRange pagedLines(File file, long position, int signedDesiredLongCount, List lines) throws IOException { + return pagedLines(file, position, signedDesiredLongCount, lines, 0); + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/util/Histotable.java b/commons/src/main/java/org/archive/util/Histotable.java new file mode 100644 index 00000000..8e28adb1 --- /dev/null +++ b/commons/src/main/java/org/archive/util/Histotable.java @@ -0,0 +1,175 @@ +/* + * Histotable.java + * + * Created on Aug 5, 2004 + * + * $Id$ + * + * Copyright (C) 2007 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + + +package org.archive.util; + +import java.util.Comparator; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + + +/** + * Collect and report frequency information. + * + * Assumes external synchronization. + * + * @author gojomo + */ +public class Histotable extends TreeMap { + private static final long serialVersionUID = 310306238032568623L; + + /** + * Record one more occurence of the given object key. + * + * @param key Object key. + */ + public void tally(K key) { + tally(key,1L); + } + + /** + * Record count more occurence(s) of the given object key. + * + * @param key Object key. + */ + public void tally(K key,long count) { + long tally = containsKey(key) ? get(key) : 0; + tally += count; + if(tally!=0) { + put(key,tally); + } else { + remove(key); + } + } + + /** + * @return Return an up-to-date sorted version of the totalled info. + */ + public TreeSet> getSortedByCounts() { + // sorted by count + TreeSet> sorted = + new TreeSet>( + new Comparator>() { + public int compare(Map.Entry e1, + Map.Entry e2) { + long firstVal = e1.getValue(); + long secondVal = e2.getValue(); + if (firstVal < secondVal) { return 1; } + if (secondVal < firstVal) { return -1; } + // If the values are the same, sort by keys. + String firstKey = (String) ((Map.Entry) e1).getKey(); + String secondKey = (String) ((Map.Entry) e2).getKey(); + return firstKey.compareTo(secondKey); + } + }); + + sorted.addAll(entrySet()); + return sorted; + } + + /** + * @return Return an up-to-date sorted version of the totalled info. + */ + public Set> getSortedByKeys() { + return entrySet(); + } + + /** + * Return the largest value of any key that is larger than 0. If no + * values or no value larger than zero, return zero. + * + * @return long largest value or zero if none larger than zero + */ + public long getLargestValue() { + long largest = 0; + for (Long el : values()) { + if (el > largest) { + largest = el; + } + } + return largest; + } + + /** + * Return the total of all tallies. + * + * @return long total of all tallies + */ + public long getTotal() { + long total = 0; + for (Long el : values()) { + total += el; + } + return total; + } + + /** + * Utility method to convert a key->Long into + * the string "count key". + * + * @param e Map key. + * @return String 'count key'. + */ + @SuppressWarnings("unchecked") + public static String entryString(Object e) { + Map.Entry entry = (Map.Entry) e; + return entry.getValue() + " " + entry.getKey(); + } + + public long add(Histotable ht) { + long net = 0; + for (K key : ht.keySet()) { + long change = ht.get(key); + net += change; + tally(key,change); + } + return net; + } + public long subtract(Histotable ht) { + long net = 0; + for (K key : ht.keySet()) { + long change = ht.get(key); + net -= change; + tally(key,-change); + } + return net; + } + + /** Return 0 instead of null for absent keys. + * + * @see java.util.TreeMap#get(java.lang.Object) + */ + @Override + public Long get(Object key) { + Long val = super.get(key); + return val == null ? 0 : val; + } + + +} diff --git a/commons/src/main/java/org/archive/util/InetAddressUtil.java b/commons/src/main/java/org/archive/util/InetAddressUtil.java new file mode 100644 index 00000000..bde3a18a --- /dev/null +++ b/commons/src/main/java/org/archive/util/InetAddressUtil.java @@ -0,0 +1,116 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.util; + +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.net.SocketException; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * InetAddress utility. + * @author stack + * @version $Date$, $Revision$ + */ +public class InetAddressUtil { + private static Logger logger = + Logger.getLogger(InetAddressUtil.class.getName()); + + /** + * ipv4 address. + */ + public static Pattern IPV4_QUADS = Pattern.compile( + "([0-9]{1,3})\\.([0-9]{1,3})\\.([0-9]{1,3})\\.([0-9]{1,3})"); + + private InetAddressUtil () { + super(); + } + + /** + * Returns InetAddress for passed host IF its in + * IPV4 quads format (e.g. 128.128.128.128). + *

TODO: Move to an AddressParsingUtil class. + * @param host Host name to examine. + * @return InetAddress IF the passed name was an IP address, else null. + */ + public static InetAddress getIPHostAddress(String host) { + InetAddress result = null; + Matcher matcher = IPV4_QUADS.matcher(host); + if (matcher == null || !matcher.matches()) { + return result; + } + try { + // Doing an Inet.getByAddress() avoids a lookup. + result = InetAddress.getByAddress(host, + new byte[] { + (byte)(new Integer(matcher.group(1)).intValue()), + (byte)(new Integer(matcher.group(2)).intValue()), + (byte)(new Integer(matcher.group(3)).intValue()), + (byte)(new Integer(matcher.group(4)).intValue())}); + } catch (NumberFormatException e) { + logger.warning(e.getMessage()); + } catch (UnknownHostException e) { + logger.warning(e.getMessage()); + } + return result; + } + + /** + * @return All known local names for this host or null if none found. + */ + public static List getAllLocalHostNames() { + List localNames = new ArrayList(); + Enumeration e = null; + try { + e = NetworkInterface.getNetworkInterfaces(); + } catch(SocketException exception) { + throw new RuntimeException(exception); + } + for (; e.hasMoreElements();) { + for (Enumeration ee = e.nextElement().getInetAddresses(); + ee.hasMoreElements();) { + InetAddress ia = ee.nextElement(); + if (ia != null) { + if (ia.getHostName() != null) { + localNames.add(ia.getHostName()); + } + if (ia.getHostAddress() != null) { + localNames.add(ia.getHostAddress()); + } + } + } + } + final String localhost = "localhost"; + if (!localNames.contains(localhost)) { + localNames.add(localhost); + } + final String localhostLocaldomain = "localhost.localdomain"; + if (!localNames.contains(localhostLocaldomain)) { + localNames.add(localhostLocaldomain); + } + return localNames; + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/util/InterruptibleCharSequence.java b/commons/src/main/java/org/archive/util/InterruptibleCharSequence.java new file mode 100644 index 00000000..dc495a53 --- /dev/null +++ b/commons/src/main/java/org/archive/util/InterruptibleCharSequence.java @@ -0,0 +1,60 @@ +/* InterruptibleCharSequence + * + * Created on Jun 27, 2007 + * + * Copyright (C) 2007 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.util; + +/** + * CharSequence that noticed thread interrupts -- as might be necessary + * to recover from a loose regex on unexpected challenging input. + * + * @author gojomo + */ +public class InterruptibleCharSequence implements CharSequence { + CharSequence inner; + // public long counter = 0; + + public InterruptibleCharSequence(CharSequence inner) { + super(); + this.inner = inner; + } + + public char charAt(int index) { + if (Thread.interrupted()) { // clears flag if set + throw new RuntimeException(new InterruptedException()); + } + // counter++; + return inner.charAt(index); + } + + public int length() { + return inner.length(); + } + + public CharSequence subSequence(int start, int end) { + return new InterruptibleCharSequence(inner.subSequence(start, end)); + } + + @Override + public String toString() { + return inner.toString(); + } +} diff --git a/commons/src/main/java/org/archive/util/InterruptibleCharSequenceTest.java b/commons/src/main/java/org/archive/util/InterruptibleCharSequenceTest.java new file mode 100644 index 00000000..69b90f28 --- /dev/null +++ b/commons/src/main/java/org/archive/util/InterruptibleCharSequenceTest.java @@ -0,0 +1,121 @@ +/* InterruptibleCharSequenceTest + * + * Created on Jun 27, 2007 + * + * Copyright (C) 2007 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.util; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.regex.Pattern; + +import junit.framework.TestCase; + +/** + * Tests (and + * @author gojomo + */ +public class InterruptibleCharSequenceTest extends TestCase { + // this regex takes many seconds to fail on the input + // (~20 seconds on 2Ghz Athlon64 JDK 1.6) + public static String BACKTRACKER = "^(((((a+)*)*)*)*)*$"; + public static String INPUT = "aaaaab"; + + /** + * Development-time benchmarking of InterruptibleCharSequence in + * regex use. (Rename 'xest' to 'test' if wanted as unit test, + * but never actually fails anything -- just measures.) + * + * For reference the regex "^(((((a+)*)*)*)*)*$" requires + * 239,286,636 charAt(s) to fail on "aaaaab", which takes + * around 20 seconds on a 2Ghz Athlon64(x2) with JDK 1.6. + * The runtime overhead of checking interrupt status in this + * extreme case is around 5% in my tests. + */ + public void xestOverhead() { + String regex = BACKTRACKER; + String inputNormal = INPUT; + InterruptibleCharSequence inputWrapped = new InterruptibleCharSequence(inputNormal); + // warm up + tryMatch(inputNormal,regex); + tryMatch(inputWrapped,regex); + // inputWrapped.counter=0; + int trials = 5; + long stringTally = 0; + long icsTally = 0; + for(int i = 1; i <= trials; i++) { + System.out.println("trial "+i+" of "+trials); + long start = System.currentTimeMillis(); + System.out.print("String "); + tryMatch(inputNormal,regex); + long end = System.currentTimeMillis(); + System.out.println(end-start); + stringTally += (end-start); + start = System.currentTimeMillis(); + System.out.print("InterruptibleCharSequence "); + tryMatch(inputWrapped,regex); + end = System.currentTimeMillis(); + System.out.println(end-start); + //System.out.println(inputWrapped.counter+" steps"); + //inputWrapped.counter=0; + icsTally += (end-start); + } + System.out.println("InterruptibleCharSequence took "+((float)icsTally)/stringTally+" longer."); + } + + public boolean tryMatch(CharSequence input, String regex) { + return Pattern.matches(regex,input); + } + + public Thread tryMatchInThread(final CharSequence input, final String regex, final BlockingQueue atFinish) { + Thread t = new Thread() { + public void run() { + boolean result; + try { + result = tryMatch(input,regex); + } catch (Exception e) { + atFinish.offer(e); + return; + } + atFinish.offer(result); + } + }; + t.start(); + return t; + } + + public void testNoninterruptible() throws InterruptedException { + BlockingQueue q = new LinkedBlockingQueue(); + Thread t = tryMatchInThread(INPUT, BACKTRACKER, q); + Thread.sleep(1000); + t.interrupt(); + Object result = q.take(); + assertTrue("mismatch uncompleted",Boolean.FALSE.equals(result)); + } + + public void testInterruptibility() throws InterruptedException { + BlockingQueue q = new LinkedBlockingQueue(); + Thread t = tryMatchInThread(new InterruptibleCharSequence(INPUT), BACKTRACKER, q); + Thread.sleep(1000); + t.interrupt(); + Object result = q.take(); + assertTrue("exception not thrown",result instanceof RuntimeException); + } +} diff --git a/commons/src/main/java/org/archive/util/Inverter.java b/commons/src/main/java/org/archive/util/Inverter.java new file mode 100644 index 00000000..5153e358 --- /dev/null +++ b/commons/src/main/java/org/archive/util/Inverter.java @@ -0,0 +1,44 @@ +/* Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Inverter.java + * Created on May 21, 2004 + * + * $Header$ + */ +package org.archive.util; + +import org.apache.commons.collections.Predicate; + + +/** + * A predicate that inverts another. + * @author gojomo + */ +public class Inverter implements Predicate { + Predicate innerPredicate; + + public Inverter(Predicate p) { + super(); + this.innerPredicate = p; + } + + public boolean evaluate(Object arg0) { + return !innerPredicate.evaluate(arg0); + } +} diff --git a/commons/src/main/java/org/archive/util/IoUtils.java b/commons/src/main/java/org/archive/util/IoUtils.java new file mode 100644 index 00000000..625406f9 --- /dev/null +++ b/commons/src/main/java/org/archive/util/IoUtils.java @@ -0,0 +1,331 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.util; + +import it.unimi.dsi.fastutil.io.FastBufferedOutputStream; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.Closeable; +import java.io.EOFException; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.OutputStream; +import java.io.Reader; +import java.io.Serializable; +import java.nio.charset.Charset; +import java.util.Iterator; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.SerializationUtils; + +/** + * I/O Utility methods. + * @author stack + * @version $Date$, $Revision$ + */ +public class IoUtils { + protected static Logger logger = + Logger.getLogger(IoUtils.class.getName()); + + /** + * @param file File to operate on. + * @return Path suitable for use getting resources off the CLASSPATH + * (CLASSPATH resources always use '/' as path separator, even on + * windows). + */ + public static String getClasspathPath(File file) { + String path = file.getPath(); + if (File.separatorChar != '/') { + // OK. We're probably on a windows system. Strip + // drive if its present and convert '\' to '/'. + path = path.replace(File.separatorChar, '/'); + int index = path.indexOf(':'); + if (index > 0 && index < 3) { + path = path.substring(index + 1); + } + } + return path; + } + + /** + * Ensure writeable directory. + * + * If doesn't exist, we attempt creation. + * + * @param dir Directory to test for exitence and is writeable. + * + * @return The passed dir. + * + * @exception IOException If passed directory does not exist and is not + * createable, or directory is not writeable or is not a directory. + */ + public static File ensureWriteableDirectory(String dir) + throws IOException { + return ensureWriteableDirectory(new File(dir)); + } + + /** + * Ensure writeable directories. + * + * If doesn't exist, we attempt creation. + * + * @param dirs List of Files to test. + * + * @return The passed dirs. + * + * @exception IOException If passed directory does not exist and is not + * createable, or directory is not writeable or is not a directory. + */ + public static List ensureWriteableDirectory(List dirs) + throws IOException { + for (Iterator i = dirs.iterator(); i.hasNext();) { + ensureWriteableDirectory(i.next()); + } + return dirs; + } + + /** + * Ensure writeable directory. + * + * If doesn't exist, we attempt creation. + * + * @param dir Directory to test for exitence and is writeable. + * + * @return The passed dir. + * + * @exception IOException If passed directory does not exist and is not + * createable, or directory is not writeable or is not a directory. + */ + public static File ensureWriteableDirectory(File dir) + throws IOException { + if (!dir.exists()) { + dir.mkdirs(); + } else { + if (!dir.canWrite()) { + throw new IOException("Dir " + dir.getAbsolutePath() + + " not writeable."); + } else if (!dir.isDirectory()) { + throw new IOException("Dir " + dir.getAbsolutePath() + + " is not a directory."); + } + } + + return dir; + } + + /** + * Read the entire stream to EOF, returning what's read as a String. + * + * @param inputStream + * @return String of the whole inputStream's contents + * @throws IOException + * @deprecated use org.apache.commons.io.IOUtils.toString() + */ + public static String readFullyAsString(InputStream inputStream) + throws IOException { + return IOUtils.toString(inputStream); + } + + + /** + * @deprecated use org.apache.commons.io.IOUtils.toString() + */ + public static String readFullyAsString(Reader r) throws IOException { + return IOUtils.toString(r); + } + + /** + * Read the entire stream to EOF into the passed file. + * @param is + * @param toFile File to read into . + * @throws IOException + * @throws IOException + */ + public static void readFullyToFile(InputStream is, + File toFile) throws IOException { + readFullyToFile(is, toFile, new byte[4096]); + } + + /** + * Read the entire stream to EOF into the passed file. + * Closes is when done or if an exception. + * @param is Stream to read. + * @param toFile File to read into . + * @param buffer Buffer to use reading. + * @return Count of bytes read. + * @throws IOException + */ + public static long readFullyToFile(final InputStream is, final File toFile, + final byte [] buffer) + throws IOException { + long totalcount = -1; + OutputStream os = + new FastBufferedOutputStream(new FileOutputStream(toFile)); + InputStream localIs = (is instanceof BufferedInputStream)? + is: new BufferedInputStream(is); + try { + for (int count = -1; + (count = localIs.read(buffer, 0, buffer.length)) != -1; + totalcount += count) { + os.write(buffer, 0, count); + } + } finally { + os.close(); + if (localIs != null) { + localIs.close(); + } + } + return totalcount; + } + + /** + * Wrap generic Throwable as a checked IOException + * @param e wrapped exception + * @return IOException + */ + public static IOException wrapAsIOException(Throwable e) { + IOException ioe = new IOException(e.toString()); + ioe.initCause(e); + return ioe; + } + + + public static void readFully(InputStream input, byte[] buf) + throws IOException { + int max = buf.length; + int ofs = 0; + while (ofs < max) { + int l = input.read(buf, ofs, max - ofs); + if (l == 0) { + throw new EOFException(); + } + ofs += l; + } + } + + + public static void close(Closeable c) { + if (c == null) { + return; + } + try { + c.close(); + } catch (IOException e) { + + } + } + + /** + * Return the maximum number of bytes per character in the named + * encoding, or 0 if encoding is invalid or unsupported. + * + * @param encoding Encoding to consider. For now, should be java + * canonical name for the encoding. + * + * @return True if multibyte encoding. + */ + public static float encodingMaxBytesPerChar(String encoding) { + boolean isMultibyte = false; + final Charset cs; + try { + if (encoding != null && encoding.length() > 0) { + cs = Charset.forName(encoding); + if(cs.canEncode()) { + return cs.newEncoder().maxBytesPerChar(); + } else { + logger.info("Encoding not fully supported: " + encoding + + ". Defaulting to single byte."); + } + } + } catch (IllegalArgumentException e) { + // Unsupported encoding + logger.log(Level.INFO,"Illegal encoding name: " + encoding,e); + } + + logger.fine("Encoding " + encoding + " is multibyte: " + + ((isMultibyte) ? Boolean.TRUE : Boolean.FALSE)); + // default: return 0 + return 0; + } + + /** + * Utility method to serialize an object to the given File. + * + * @param object Object to serialize + * @param file File to receive serialized copy + * @throws IOException + */ + public static void serializeToFile(Object object, File file) throws IOException { + ObjectOutputStream oos = new ObjectOutputStream(new BufferedOutputStream(new FileOutputStream(file))); + oos.writeObject(object); + oos.close(); + } + + /** + * Utility method to deserialize an Object from given File. + * + * @param file File source + * @return deserialized Object + * @throws IOException + */ + public static Object deserializeFromFile(File file) throws IOException { + ObjectInputStream ois = new ObjectInputStream(new BufferedInputStream(new FileInputStream(file))); + Object object; + try { + object = ois.readObject(); + } catch (ClassNotFoundException e) { + // TODO Auto-generated catch block + throw new RuntimeException(e); + } + ois.close(); + return object; + } + + /** + * Utility method to serialize Object to byte[]. + * + * @param object Object to be serialized + * @return byte[] serialized form + * @deprecated use SerializationUtils.serialize + */ + public static byte[] serializeToByteArray(Object object) { + return SerializationUtils.serialize((Serializable)object); + } + + /** + * Utility method to deserialize Object from byte[]. + * + * @param in byte[] source + * @return Object deserialized + * @deprecated use SerializationUtils.deserialize + */ + public static Object deserializeFromByteArray(byte[] in) { + return SerializationUtils.deserialize(in); + } +} diff --git a/commons/src/main/java/org/archive/util/JavaLiterals.java b/commons/src/main/java/org/archive/util/JavaLiterals.java new file mode 100644 index 00000000..3059afd6 --- /dev/null +++ b/commons/src/main/java/org/archive/util/JavaLiterals.java @@ -0,0 +1,148 @@ +/* JavaLiterals + * + * Created on Dec 31, 2003 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.util; + +/** + * Utility functions to escape or unescape Java literal strings. + * + * @author gojomo + * + */ +public class JavaLiterals { + + public static String escape(String raw) { + StringBuffer escaped = new StringBuffer(); + for(int i = 0; i=escaped.length()) { + // trailing '/' + raw.append(c); + continue; + } + c = escaped.charAt(i); + switch (c) { + case 'b': + raw.append('\b'); + break; + case 't': + raw.append('\t'); + break; + case 'n': + raw.append('\n'); + break; + case 'f': + raw.append('\f'); + break; + case 'r': + raw.append('r'); + break; + case '"': + raw.append('\"'); + break; + case '\'': + raw.append('\''); + break; + case '\\': + raw.append('\\'); + break; + case 'u': + // unicode hex escape + try { + int unicode = Integer.parseInt(escaped.substring(i+1,i+5),16); + raw.append((char)unicode); + i = i + 4; + } catch (IndexOutOfBoundsException e) { + // err + raw.append("\\u"); + } + break; + default: + if(Character.isDigit(c)) { + // octal escape + int end = Math.min(i+4,escaped.length()); + int octal = Integer.parseInt(escaped.substring(i+1,end),8); + if(octal<256) { + raw.append((char)octal); + i = end - 1; + } else { + // err + raw.append('\\'); + raw.append(c); + } + } + break; + } + } + } + return raw.toString(); + } +} diff --git a/commons/src/main/java/org/archive/util/JndiUtils.java b/commons/src/main/java/org/archive/util/JndiUtils.java new file mode 100644 index 00000000..63d985b8 --- /dev/null +++ b/commons/src/main/java/org/archive/util/JndiUtils.java @@ -0,0 +1,182 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.util; + +import java.util.Enumeration; +import java.util.Hashtable; +import java.util.Properties; + +import javax.management.MalformedObjectNameException; +import javax.management.ObjectName; +import javax.naming.CompoundName; +import javax.naming.Context; +import javax.naming.InitialContext; +import javax.naming.InvalidNameException; +import javax.naming.NameNotFoundException; +import javax.naming.NamingException; +import javax.naming.RefAddr; +import javax.naming.Reference; +import javax.naming.StringRefAddr; + +/** + * JNDI utilities. + * @author stack + * @version $Date$ $Version$ + */ +public class JndiUtils { + /** + * Syntax that will work with jmx ObjectNames (i.e. will escape '.' and + * will add treat ',' and '=' specially. + */ + private static final Properties COMPOUND_NAME_SYNTAX = new Properties(); + static { + COMPOUND_NAME_SYNTAX.put("jndi.syntax.direction", "left_to_right"); + COMPOUND_NAME_SYNTAX.put("jndi.syntax.separator", "+"); + COMPOUND_NAME_SYNTAX.put("jndi.syntax.ignorecase", "false"); + COMPOUND_NAME_SYNTAX.put("jndi.syntax.escape", "\\"); + COMPOUND_NAME_SYNTAX.put("jndi.syntax.beginquote", "'"); + COMPOUND_NAME_SYNTAX.put("jndi.syntax.trimblanks", "true"); + COMPOUND_NAME_SYNTAX.put("jndi.syntax.separator.ava", ","); + COMPOUND_NAME_SYNTAX.put("jndi.syntax.separator.typeval", "="); + } + + public static CompoundName getCompoundName(final String name) + throws InvalidNameException { + return new CompoundName(name, COMPOUND_NAME_SYNTAX); + } + + /** + * Return name to use as jndi name. + * Used to do a subset of the ObjectName fields but not just + * let all through so its easy to just use the jndi name to + * find mbean. + * @param on ObjectName instance to work with. + * @return Returns a compound name to use as jndi key. + * @throws NullPointerException + * @throws InvalidNameException + */ + public static CompoundName getCompoundName(final ObjectName on) + throws NullPointerException, + InvalidNameException { + return getCompoundName(on.getCanonicalKeyPropertyListString()); + } + + /** + * @param on ObjectName instance to work with. + * @return A simple reference based on passed on + */ + public static Reference getReference(final ObjectName on) { + Reference r = new Reference(String.class.getName()); + Hashtable ht = on.getKeyPropertyList(); + r.add(new StringRefAddr("host", (String)ht.get("host"))); + r.add(new StringRefAddr("name", (String)ht.get("name"))); + // Put in a value to serve as a unique 'key'. + r.add(new StringRefAddr("key", + on.getCanonicalKeyPropertyListString())); + return r; + } + + /** + * Get subcontext. Only looks down one level. + * @param subContext Name of subcontext to return. + * @return Sub context. + * @throws NamingException + */ + public static Context getSubContext(final String subContext) + throws NamingException { + return getSubContext(getCompoundName(subContext)); + } + + /** + * Get subcontext. Only looks down one level. + * @param subContext Name of subcontext to return. + * @return Sub context. + * @throws NamingException + */ + public static Context getSubContext(final CompoundName subContext) + throws NamingException { + Context context = new InitialContext(); + try { + context = (Context)context.lookup(subContext); + } catch (NameNotFoundException e) { + context = context.createSubcontext(subContext); + } + return context; + } + + /** + * + * @param context A subcontext named for the on.getDomain() + * (Assumption is that caller already setup this subcontext). + * @param on The ObjectName we're to base our bind name on. + * @return Returns key we used binding this ObjectName. + * @throws NamingException + * @throws NullPointerException + */ + public static CompoundName bindObjectName(Context context, + final ObjectName on) + throws NamingException, NullPointerException { + // I can't call getNameInNamespace in tomcat. Complains about + // unsupported operation -- that I can't get absolute name. + // Therefore just skip this test below -- at least for now. + // Check that passed context has the passed ObjectNames' name. + // +// String name = getCompoundName(context.getNameInNamespace()).toString(); +// if (!name.equals(on.getDomain())) { +// throw new NamingException("The current context is " + name + +// " but domain is " + on.getDomain() + " (Was expecting " + +// "them to be the same)."); +// } + CompoundName key = getCompoundName(on); + context.rebind(key, getReference(on)); + return key; + } + + public static CompoundName unbindObjectName(final Context context, + final ObjectName on) + throws NullPointerException, NamingException { + CompoundName key = getCompoundName(on); + context.unbind(key); + return key; + } + + + /** + * Testing code. + * @param args Command line arguments. + * @throws NullPointerException + * @throws MalformedObjectNameException + * @throws NamingException + * @throws InvalidNameException + */ + public static void main(String[] args) + throws MalformedObjectNameException, NullPointerException, + InvalidNameException, NamingException { + final ObjectName on = new ObjectName("org.archive.crawler:" + + "type=Service,name=Heritrix00,host=debord.archive.org"); + Context c = getSubContext(getCompoundName(on.getDomain())); + CompoundName key = bindObjectName(c, on); + Reference r = (Reference)c.lookup(key); + for (Enumeration e = r.getAll(); e.hasMoreElements();) { + System.out.println(e.nextElement()); + } + unbindObjectName(c, on); + } +} diff --git a/commons/src/main/java/org/archive/util/LRU.java b/commons/src/main/java/org/archive/util/LRU.java new file mode 100644 index 00000000..b441b510 --- /dev/null +++ b/commons/src/main/java/org/archive/util/LRU.java @@ -0,0 +1,69 @@ +/* LRU +* +* Created on September 18, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util; + +import java.util.LinkedHashMap; +import java.util.Map; + + +/** + * A least-recently used cache. As new entries are added to the map, the + * least-recently accessed entries are removed. + * + * @author pjack + * + * @param The key type of the LRU + * @param The value type of the LRU + */ +public class LRU extends LinkedHashMap { + + + /** + * Generated by Eclipse. + */ + private static final long serialVersionUID = 1032420936705267913L; + + + /** + * The maximum number of entries to store in the cache. + */ + private int max; + + + /** + * Constructor. + * + * @param max the maximum number of entries to cache + */ + public LRU(int max) { + super(max, (float)0.75, true); + this.max = max; + } + + + @Override + protected boolean removeEldestEntry(Map.Entry entry) { + return size() >= max; + } + +} diff --git a/commons/src/main/java/org/archive/util/LongWrapper.java b/commons/src/main/java/org/archive/util/LongWrapper.java new file mode 100644 index 00000000..ca685eb8 --- /dev/null +++ b/commons/src/main/java/org/archive/util/LongWrapper.java @@ -0,0 +1,42 @@ +/* LongWrapper + * + * $Id$ + * + * Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.util; + +import java.io.Serializable; + +/** + * Wraps a long. Used in place of Long so that when we extract it from a + * Collection we can modify the long value without creating a new object. + * This way we don't have to rewrite the Collection to update one of the + * stored longs. + * @author Kristinn Sigurdsson + */ +public class LongWrapper implements Serializable { + + private static final long serialVersionUID = -6537350490019555280L; + + public long longValue; + public LongWrapper(long initial){ + this.longValue = initial; + } +} diff --git a/commons/src/main/java/org/archive/util/MimetypeUtils.java b/commons/src/main/java/org/archive/util/MimetypeUtils.java new file mode 100644 index 00000000..7866b000 --- /dev/null +++ b/commons/src/main/java/org/archive/util/MimetypeUtils.java @@ -0,0 +1,81 @@ +/* MimetypeUtils + * + * $Id$ + * + * Created on Sep 22, 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.util; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Class of mimetype utilities. + * @author stack + */ +public class MimetypeUtils { + /** + * The 'no-type' content-type. + * + * Defined in the ARC file spec at + * http://www.archive.org/web/researcher/ArcFileFormat.php. + */ + public static final String NO_TYPE_MIMETYPE = "no-type"; + + /** + * Truncation regex. + */ + final static Pattern TRUNCATION_REGEX = Pattern.compile("^([^\\s;,]+).*"); + + + /** + * Truncate passed mimetype. + * + * Ensure no spaces. Strip encoding. Truncation required by + * ARC files. + * + *

Truncate at delimiters [;, ]. + * Truncate multi-part content type header at ';'. + * Apache httpclient collapses values of multiple instances of the + * header into one comma-separated value,therefore truncated at ','. + * Current ia_tools that work with arc files expect 5-column + * space-separated meta-lines, therefore truncate at ' '. + * + * @param contentType Raw content-type. + * + * @return Computed content-type made from passed content-type after + * running it through a set of rules. + */ + public static String truncate(String contentType) { + if (contentType == null) { + contentType = NO_TYPE_MIMETYPE; + } else { + Matcher matcher = TRUNCATION_REGEX.matcher(contentType); + if (matcher.matches()) { + contentType = matcher.group(1); + } else { + contentType = NO_TYPE_MIMETYPE; + } + } + + return contentType; + } +} diff --git a/commons/src/main/java/org/archive/util/MultiReporter.java b/commons/src/main/java/org/archive/util/MultiReporter.java new file mode 100644 index 00000000..9190ca08 --- /dev/null +++ b/commons/src/main/java/org/archive/util/MultiReporter.java @@ -0,0 +1,46 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.util; + +import java.io.PrintWriter; + +/** + * @contributor stack + */ +public interface MultiReporter extends Reporter { + /** + * Get an array of report names offered by this Reporter. + * A name in brackets indicates a free-form String, + * in accordance with the informal description inside + * the brackets, may yield a useful report. + * + * @return String array of report names, empty if there is only + * one report type + */ + public String[] getReports(); + + /** + * Make a report of the given name to the passed-in Writer, + * If null, give the default report. + * + * @param writer to receive report + */ + public void reportTo(String name, PrintWriter writer); +} diff --git a/commons/src/main/java/org/archive/util/OneLineSimpleLogger.java b/commons/src/main/java/org/archive/util/OneLineSimpleLogger.java new file mode 100644 index 00000000..2cd7fe0b --- /dev/null +++ b/commons/src/main/java/org/archive/util/OneLineSimpleLogger.java @@ -0,0 +1,137 @@ +/* OneLineSimpleLogger + * + * $Id$ + * + * Created on Jul 22, 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.util; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.text.FieldPosition; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.logging.ConsoleHandler; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; +import java.util.logging.SimpleFormatter; + + + +/** + * Logger that writes entry on one line with less verbose date. + * + * @author stack + * @version $Revision$, $Date$ + */ +public class OneLineSimpleLogger extends SimpleFormatter { + + /** + * Date instance. + * + * Keep around instance of date. + */ + private Date date = new Date(); + + /** + * Field position instance. + * + * Keep around this instance. + */ + private FieldPosition position = new FieldPosition(0); + + /** + * MessageFormatter for date. + */ + private SimpleDateFormat formatter = + new SimpleDateFormat("MM/dd/yyyy HH:mm:ss Z"); + + /** + * Persistent buffer in which we conjure the log. + */ + private StringBuffer buffer = new StringBuffer(); + + + public OneLineSimpleLogger() { + super(); + } + + public synchronized String format(LogRecord record) { + this.buffer.setLength(0); + this.date.setTime(record.getMillis()); + this.position.setBeginIndex(0); + this.formatter.format(this.date, buffer, this.position); + buffer.append(' '); + buffer.append(record.getLevel().getLocalizedName()); + buffer.append(' '); + if (record.getSourceClassName() != null) { + buffer.append(record.getSourceClassName()); + } else { + buffer.append(record.getLoggerName()); + } + buffer.append(' '); + String methodName = record.getSourceMethodName(); + methodName = (methodName == null || methodName.length() <= 0)? + "-": methodName; + buffer.append(methodName); + buffer.append(' '); + buffer.append(formatMessage(record)); + buffer.append(System.getProperty("line.separator")); + if (record.getThrown() != null) { + try { + StringWriter writer = new StringWriter(); + PrintWriter printer = new PrintWriter(writer); + record.getThrown().printStackTrace(printer); + writer.close(); + buffer.append(writer.toString()); + } catch (Exception e) { + buffer.append("Failed to get stack trace: " + e.getMessage()); + } + } + return buffer.toString(); + } + + public static Logger setConsoleHandler() { + Logger logger = Logger.getLogger(""); + Handler [] hs = logger.getHandlers(); + for (int i = 0; i < hs.length; i++) { + Handler h = hs[0]; + if (h instanceof ConsoleHandler) { + h.setFormatter(new OneLineSimpleLogger()); + } + } + return logger; + } + + /** + * Test this logger. + */ + public static void main(String[] args) { + Logger logger = setConsoleHandler(); + logger = Logger.getLogger("Test"); + logger.severe("Does this come out?"); + logger.severe("Does this come out?"); + logger.severe("Does this come out?"); + logger.log(Level.SEVERE, "hello", new RuntimeException("test")); + } +} diff --git a/commons/src/main/java/org/archive/util/PaddingStringBuffer.java b/commons/src/main/java/org/archive/util/PaddingStringBuffer.java new file mode 100644 index 00000000..626d1937 --- /dev/null +++ b/commons/src/main/java/org/archive/util/PaddingStringBuffer.java @@ -0,0 +1,170 @@ +/* Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * PaddingStringBuffer.java + * Created on Oct 23, 2003 + * + * $Header$ + */ +package org.archive.util; + +/** + * StringBuffer-like utility which can add spaces to reach a certain column. It + * allows you to append {@link String}, long and ints + * to the buffer. + *

+ * Note: This class counts from 1, not 0. + *

+ * It uses a StringBuffer behind the scenes. + *

+ * To write a string with multiple lines, it is advisible to use the + * {@link #newline() newline()} function. Regular appending of strings with + * newlines (\n) character should be safe though. Right appending of strings + * with such characters is not safe. + * + * @author Gordon Mohr + */ +public final class PaddingStringBuffer { + // The buffer. + StringBuffer buffer; + // Location in current line + int linePos; + + /** + * Create a new PaddingStringBuffer + * + */ + public PaddingStringBuffer() { + buffer = new StringBuffer(); + linePos=0; + } + + /** append a string directly to the buffer + * @param string the string to append + * @return This wrapped buffer w/ the passed string appended. + */ + public PaddingStringBuffer append(String string) { + buffer.append(string); + if ( string.indexOf('\n') == -1 ){ + linePos+=string.length(); + } else { + while ( string.indexOf('\n') == -1 ){ + string = string.substring(string.indexOf('\n')); + } + linePos=string.length(); + } + return this; + } + + /** + * Append a string, right-aligned to the given columm. If the buffer + * length is already greater than the column specified, it simply appends + * the string + * + * @param col the column to right-align to + * @param string the string, must not contain multiple lines. + * @return This wrapped buffer w/ append string, right-aligned to the + * given column. + */ + public PaddingStringBuffer raAppend(int col, String string) { + padTo(col-string.length()); + append(string); + return this; + } + + /** Pad to a given column. If the buffer size is already greater than the + * column, nothing is done. + * @param col + * @return The buffer padded to i. + */ + public PaddingStringBuffer padTo(int col) { + while(linePosint to the buffer. + * @param i the int to append + * @return This wrapped buffer with i appended. + */ + public PaddingStringBuffer append(int i) { + append(Integer.toString(i)); + return this; + } + + + /** + * Append an int right-aligned to the given column. If the + * buffer length is already greater than the column specified, it simply + * appends the int. + * + * @param col the column to right-align to + * @param i the int to append + * @return This wrapped buffer w/ appended int, right-aligned to the + * given column. + */ + public PaddingStringBuffer raAppend(int col, int i) { + return raAppend(col,Integer.toString(i)); + } + + /** append a long to the buffer. + * @param lo the long to append + * @return This wrapped buffer w/ appended long. + */ + public PaddingStringBuffer append(long lo) { + append(Long.toString(lo)); + return this; + } + + /**Append a long, right-aligned to the given column. If the + * buffer length is already greater than the column specified, it simply + * appends the long. + * @param col the column to right-align to + * @param lo the long to append + * @return This wrapped buffer w/ appended long, right-aligned to the + * given column. + */ + public PaddingStringBuffer raAppend(int col, long lo) { + return raAppend(col,Long.toString(lo)); + } + + /** reset the buffer back to empty */ + public void reset() { + buffer = new StringBuffer(); + linePos = 0; + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + public String toString() { + return buffer.toString(); + } + + /** + * Forces a new line in the buffer. + */ + public PaddingStringBuffer newline() { + buffer.append("\n"); + linePos = 0; + return this; + } + +} diff --git a/commons/src/main/java/org/archive/util/PatternMatcherRecycler.java b/commons/src/main/java/org/archive/util/PatternMatcherRecycler.java new file mode 100644 index 00000000..3c37ed9a --- /dev/null +++ b/commons/src/main/java/org/archive/util/PatternMatcherRecycler.java @@ -0,0 +1,91 @@ +/* PatternMatcherRecycler +* +* $Id$ +* +* Created on Dec 21, 2004 +* +* Copyright (C) 2004 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util; + +import java.util.EmptyStackException; +import java.util.Stack; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Utility class to retain a compiled Pattern and multiple corresponding + * Matcher instances for reuse. + * + * @author gojomo + */ +public class PatternMatcherRecycler { + /** + * Upper-bound on Matcher Stacks. + * Profiling has the size of these Stacks tending upward over + * the life of a crawl. TODO: do something better than an + * a coarse upperbound; do something that can get GC'd in + * low-memory conditions. + */ + private final static int MAXIMUM_STACK_SIZE = 10; + + private Pattern pattern; + private Stack matchers; + + public PatternMatcherRecycler(Pattern p) { + this.pattern = p; + this.matchers = new Stack(); + } + + public Pattern getPattern() { + return this.pattern; + } + + /** + * Get a Matcher for the internal Pattern, against the given + * input sequence. Reuse an old Matcher if possible, otherwise + * create a new one. + * + * @param input CharSequence to match + * @return Matcher set against the the input sequence + */ + public Matcher getMatcher(CharSequence input) { + if (input == null) { + throw new IllegalArgumentException("CharSequence 'input' must not be null"); + } + try { + return ((Matcher)matchers.pop()).reset(input); + } catch (EmptyStackException e) { + return this.pattern.matcher(input); + } + } + + /** + * Return the given Matcher to the reuse stack, if stack is + * not already at its maximum size. + * + * @param m the Matcher to save for reuse + */ + public void freeMatcher(Matcher m) { + if(this.matchers.size() < MAXIMUM_STACK_SIZE) { + matchers.push(m); + } + } +} + diff --git a/commons/src/main/java/org/archive/util/PreJ15Utils.java b/commons/src/main/java/org/archive/util/PreJ15Utils.java new file mode 100644 index 00000000..ab557023 --- /dev/null +++ b/commons/src/main/java/org/archive/util/PreJ15Utils.java @@ -0,0 +1,50 @@ +/* PreJ15Utils +* +* $Id$ +* +* Created on May 20, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util; + +/** + * A collection of utility methods doing things that are easier in Java 1.5. + * + * @author gojomo + * @deprecated Will be removed post 1.10.0 Heritrix. + */ +public class PreJ15Utils { + + /** + * Version of 1.5's StringBuffer.append(CharSequence s, int start, int finish) + * @param buffer StringBuffer to append to + * @param cs CharSequence with material to append + * @param start position from which to begin appending + * @param end position at which to stop appending (exclusive) + */ + public static StringBuffer append(StringBuffer buffer, CharSequence cs, int start, int end) { + // in 1.5, this would be builder.append(cs, start, end); + for(int i = start; iPut another way, for every element in the result list, the following + * expression will be true: string.startsWith(element.getKey()). + * + * @param set the sorted set containing potential prefixes + * @param input the string whose prefixes to find + * @return the list of prefixes + */ + public static List find(SortedSet set, String input) { + LinkedList result = new LinkedList(); + set = headSetInclusive(set, input); + int opCount = 0; + for (String last = last(set); last != null; last = last(set)) { + opCount++; + if (input.startsWith(last)) { + result.push(last); + set = set.headSet(last); + } else { + // Find the longest common prefix. + int p = StringUtils.indexOfDifference(input, last); + if (p <= 0) { + return result; + } + last = input.substring(0, p); + set = headSetInclusive(set, last); + } + } + return result; + } + + + @SuppressWarnings("unchecked") + protected static SortedSet headSetInclusive(SortedSet set, String input) { + // use NavigableSet inclusive version if available + if(set instanceof NavigableSet) { + return ((NavigableSet)set).headSet(input, true); + } + // use Stored*Set inclusive version if available + if(set instanceof StoredSortedKeySet) { + return ((StoredSortedKeySet)set).headSet(input, true); + } + if(set instanceof StoredSortedValueSet) { + return ((StoredSortedValueSet)set).headSet(input, true); + } + // Use synthetic "one above" trick + // NOTE: because '\0' sorts in the middle in "java modified UTF-8", + // used in the Stored* class StringBindings, this trick won't work + // there + return set.headSet(input+'\0'); + } + + + private static String last(SortedSet set) { + return set.isEmpty() ? null : set.last(); + } + + + public static List findKeys(SortedMap map, String input) { + LinkedList result = new LinkedList(); + map = headMapInclusive(map, input); + int opCount = 0; + for (String last = last(map); last != null; last = last(map)) { + opCount++; + if (input.startsWith(last)) { + result.push(last); + map = map.headMap(last); + } else { + // Find the longest common prefix. + int p = StringUtils.indexOfDifference(input, last); + if (p <= 0) { + return result; + } + last = input.substring(0, p); + map = headMapInclusive(map, last); + } + } + return result; + } + + + @SuppressWarnings("unchecked") + private static SortedMap headMapInclusive(SortedMap map, String input) { + // use NavigableMap inclusive version if available + if(map instanceof NavigableMap) { + return ((NavigableMap)map).headMap(input, true); + } + // use StoredSortedMap inclusive version if available + if(map instanceof StoredSortedMap) { + return ((StoredSortedMap)map).headMap(input, true); + } + // Use synthetic "one above" trick + // NOTE: because '\0' sorts in the middle in "java modified UTF-8", + // used in the Stored* class StringBindings, this trick won't work + // there + return map.headMap(input+'\0'); + } + + + private static String last(SortedMap map) { + // TODO Auto-generated method stub + return map.isEmpty() ? null : map.lastKey(); + } +} diff --git a/commons/src/main/java/org/archive/util/PrefixSet.java b/commons/src/main/java/org/archive/util/PrefixSet.java new file mode 100644 index 00000000..e5fe7fd8 --- /dev/null +++ b/commons/src/main/java/org/archive/util/PrefixSet.java @@ -0,0 +1,74 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.util; + +import java.util.SortedSet; +import java.util.TreeSet; + +/** + * Utility class for maintaining sorted set of string prefixes. + * Redundant prefixes are coalesced into the shorter prefix. + */ +public class PrefixSet extends TreeSet { + private static final long serialVersionUID = -6054697706348411992L; + + public PrefixSet() { + super(); + } + + /** + * Test whether the given String is prefixed by one + * of this set's entries. + * + * @param s + * @return True if contains prefix. + */ + public boolean containsPrefixOf(String s) { + SortedSet sub = headSet(s); + // because redundant prefixes have been eliminated, + // only a test against last item in headSet is necessary + if (!sub.isEmpty() && s.startsWith((String)sub.last())) { + return true; // prefix substring exists + } // else: might still exist exactly (headSet does not contain boundary) + return contains(s); // exact string exists, or no prefix is there + } + + /** + * Maintains additional invariant: if one entry is a + * prefix of another, keep only the prefix. + * + * @see java.util.Collection#add(java.lang.Object) + */ + public boolean add(String s) { + SortedSet sub = headSet(s); + if (!sub.isEmpty() && s.startsWith((String)sub.last())) { + // no need to add; prefix is already present + return false; + } + boolean retVal = super.add(s); + sub = tailSet(s+"\0"); + while(!sub.isEmpty() && ((String)sub.first()).startsWith(s)) { + // remove redundant entries + sub.remove(sub.first()); + } + return retVal; + } + +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/util/ProcessUtils.java b/commons/src/main/java/org/archive/util/ProcessUtils.java new file mode 100644 index 00000000..74b2bd86 --- /dev/null +++ b/commons/src/main/java/org/archive/util/ProcessUtils.java @@ -0,0 +1,156 @@ +/* ProcessUtils.java + * + * $Id$ + * + * Created Jul 19, 2005 + * + * Copyright (C) 2005 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.util; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Class to run an external process. + * @author stack + * @version $Date$ $Revision$ + */ +public class ProcessUtils { + private static final Logger LOGGER = + Logger.getLogger(ProcessUtils.class.getName()); + + protected ProcessUtils() { + super(); + } + + /** + * Thread to gobble up an output stream. + * See http://www.javaworld.com/javaworld/jw-12-2000/jw-1229-traps.html + */ + protected class StreamGobbler extends Thread { + private final InputStream is; + private final StringBuffer sink = new StringBuffer(); + + StreamGobbler(InputStream is, String name) { + this.is = is; + setName(name); + } + + public void run() { + try { + BufferedReader br = + new BufferedReader(new InputStreamReader(this.is)); + for (String line = null; (line = br.readLine()) != null;) { + this.sink.append(line); + } + } catch (IOException ioe) { + ioe.printStackTrace(); + } + } + + public String getSink() { + return this.sink.toString(); + } + } + + /** + * Data structure to hold result of a process exec. + * @author stack + * @version $Date$ $Revision$ + */ + public class ProcessResult { + private final String [] args; + private final int result; + private final String stdout; + private final String stderr; + + protected ProcessResult(String [] args, int result, String stdout, + String stderr) { + this.args = args; + this.result = result; + this.stderr = stderr; + this.stdout = stdout; + } + + public int getResult() { + return this.result; + } + + public String getStdout() { + return this.stdout; + } + + public String getStderr() { + return this.stderr; + } + + public String toString() { + StringBuffer sb = new StringBuffer(); + for (int i = 0; i < this.args.length; i++) { + sb.append(this.args[i]); + sb.append(", "); + } + return sb.toString() + " exit code: " + this.result + + ((this.stderr != null && this.stderr.length() > 0)? + "\nSTDERR: " + this.stderr: "") + + ((this.stdout != null && this.stdout.length() > 0)? + "\nSTDOUT: " + this.stdout: ""); + } + } + + /** + * Runs process. + * @param args List of process args. + * @return A ProcessResult data structure. + * @throws IOException If interrupted, we throw an IOException. If non-zero + * exit code, we throw an IOException (This may need to change). + */ + public static ProcessUtils.ProcessResult exec(String [] args) + throws IOException { + Process p = Runtime.getRuntime().exec(args); + ProcessUtils pu = new ProcessUtils(); + // Gobble up any output. + StreamGobbler err = pu.new StreamGobbler(p.getErrorStream(), "stderr"); + err.setDaemon(true); + err.start(); + StreamGobbler out = pu.new StreamGobbler(p.getInputStream(), "stdout"); + out.setDaemon(true); + out.start(); + int exitVal; + try { + exitVal = p.waitFor(); + } catch (InterruptedException e) { + throw new IOException("Wait on process " + args + " interrupted: " + + e.getMessage()); + } + ProcessUtils.ProcessResult result = + pu.new ProcessResult(args, exitVal, out.getSink(), err.getSink()); + if (exitVal != 0) { + throw new IOException(result.toString()); + } else if (LOGGER.isLoggable(Level.INFO)) { + LOGGER.info(result.toString()); + } + return result; + } +} diff --git a/commons/src/main/java/org/archive/util/ProgressStatisticsReporter.java b/commons/src/main/java/org/archive/util/ProgressStatisticsReporter.java new file mode 100644 index 00000000..69ef9d00 --- /dev/null +++ b/commons/src/main/java/org/archive/util/ProgressStatisticsReporter.java @@ -0,0 +1,42 @@ +/* StatisticsReporter.java + * + * $Id$ + * + * Created Jul 18, 2005 + * + * Copyright (C) 2005 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.util; + +import java.io.IOException; +import java.io.PrintWriter; + +public interface ProgressStatisticsReporter { + /** + * @param writer Where to write statistics. + * @throws IOException + */ + public void progressStatisticsLine(PrintWriter writer) throws IOException; + + /** + * @param writer Where to write statistics legend. + * @throws IOException + */ + public void progressStatisticsLegend(PrintWriter writer) throws IOException; +} diff --git a/commons/src/main/java/org/archive/util/PropertyUtils.java b/commons/src/main/java/org/archive/util/PropertyUtils.java new file mode 100644 index 00000000..180b9c65 --- /dev/null +++ b/commons/src/main/java/org/archive/util/PropertyUtils.java @@ -0,0 +1,58 @@ +/* PropertyUtils.java + * + * Created Aug 4, 2005 + * + * Copyright (C) 2005 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.util; + +/** + * @author stack + * @version $Date$ $Revision$ + */ +public class PropertyUtils { + /*** + * @param key Property key. + * @return Named property or null if the property is null or empty. + */ + public static String getPropertyOrNull(final String key) { + String value = System.getProperty(key); + return (value == null || value.length() <= 0)? null: value; + } + + /*** + * @param key Property key. + * @return Boolean value or false if null or unreadable. + */ + public static boolean getBooleanProperty(final String key) { + return (getPropertyOrNull(key) == null)? + false: Boolean.valueOf(getPropertyOrNull(key)).booleanValue(); + } + + /** + * @param key Key to use looking up system property. + * @param fallback If no value found for passed key, return + * fallback. + * @return Value of property or fallback. + */ + public static int getIntProperty(final String key, final int fallback) { + return getPropertyOrNull(key) == null? + fallback: Integer.parseInt(getPropertyOrNull(key)); + } +} diff --git a/commons/src/main/java/org/archive/util/Recorder.java b/commons/src/main/java/org/archive/util/Recorder.java new file mode 100644 index 00000000..aff76e93 --- /dev/null +++ b/commons/src/main/java/org/archive/util/Recorder.java @@ -0,0 +1,342 @@ +/* HTTPRecorder + * + * $Id$ + * + * Created on Sep 22, 2003 + * + * Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.util; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.archive.io.RecordingInputStream; +import org.archive.io.RecordingOutputStream; +import org.archive.io.ReplayCharSequence; +import org.archive.io.ReplayInputStream; + + +/** + * Pairs together a RecordingInputStream and RecordingOutputStream + * to capture exactly a single HTTP transaction. + * + * Initially only supports HTTP/1.0 (one request, one response per stream) + * + * Call {@link #markContentBegin()} to demarc the transition between HTTP + * header and body. + * + * @author gojomo + */ +public class Recorder { + protected static Logger logger = + Logger.getLogger("org.archive.util.HttpRecorder"); + + private static final int DEFAULT_OUTPUT_BUFFER_SIZE = 4096; + private static final int DEFAULT_INPUT_BUFFER_SIZE = 65536; + + private RecordingInputStream ris = null; + private RecordingOutputStream ros = null; + + /** + * Backing file basename. + * + * Keep it around so can clean up backing files left on disk. + */ + private String backingFileBasename = null; + + /** + * Backing file output stream suffix. + */ + private static final String RECORDING_OUTPUT_STREAM_SUFFIX = ".ros"; + + /** + * Backing file input stream suffix. + */ + private static final String RECORDING_INPUT_STREAM_SUFFIX = ".ris"; + + /** + * Response character encoding. + */ + private String characterEncoding = null; + + + /** + * Create an HttpRecorder. + * + * @param tempDir Directory into which we drop backing files for + * recorded input and output. + * @param backingFilenameBase Backing filename base to which we'll append + * suffices ris for recorded input stream and + * ros for recorded output stream. + * @param outBufferSize Size of output buffer to use. + * @param inBufferSize Size of input buffer to use. + */ + public Recorder(File tempDir, String backingFilenameBase, + int outBufferSize, int inBufferSize) { + this(ensure(new File(tempDir, backingFilenameBase)), + outBufferSize, inBufferSize); + tempDir.mkdirs(); + } + + + private static File ensure(File tempDir) { + tempDir.mkdirs(); + return tempDir; + } + + public Recorder(File file, int outBufferSize, int inBufferSize) { + super(); + this.backingFileBasename = file.getAbsolutePath(); + this.ris = new RecordingInputStream(inBufferSize, + this.backingFileBasename + RECORDING_INPUT_STREAM_SUFFIX); + this.ros = new RecordingOutputStream(outBufferSize, + this.backingFileBasename + RECORDING_OUTPUT_STREAM_SUFFIX); + } + + /** + * Create an HttpRecorder. + * + * @param tempDir + * Directory into which we drop backing files for recorded input + * and output. + * @param backingFilenameBase + * Backing filename base to which we'll append suffices + * ris for recorded input stream and + * ros for recorded output stream. + */ + public Recorder(File tempDir, String backingFilenameBase) { + this(tempDir, backingFilenameBase, DEFAULT_INPUT_BUFFER_SIZE, + DEFAULT_OUTPUT_BUFFER_SIZE); + } + + + /** + * Wrap the provided stream with the internal RecordingInputStream + * + * open() throws an exception if RecordingInputStream is already open. + * + * @param is InputStream to wrap. + * + * @return The input stream wrapper which itself is an input stream. + * Pass this in place of the passed stream so input can be recorded. + * + * @throws IOException + */ + public InputStream inputWrap(InputStream is) + throws IOException { + logger.fine(Thread.currentThread().getName() + " wrapping input"); + this.ris.open(is); + return this.ris; + } + + /** + * Wrap the provided stream with the internal RecordingOutputStream + * + * open() throws an exception if RecordingOutputStream is already open. + * + * @param os The output stream to wrap. + * + * @return The output stream wrapper which is itself an output stream. + * Pass this in place of the passed stream so output can be recorded. + * + * @throws IOException + */ + public OutputStream outputWrap(OutputStream os) + throws IOException { + this.ros.open(os); + return this.ros; + } + + /** + * Close all streams. + */ + public void close() { + logger.fine(Thread.currentThread().getName() + " closing"); + try { + this.ris.close(); + } catch (IOException e) { + // TODO: Can we not let the exception out of here and report it + // higher up in the caller? + DevUtils.logger.log(Level.SEVERE, "close() ris" + + DevUtils.extraInfo(), e); + } + try { + this.ros.close(); + } catch (IOException e) { + DevUtils.logger.log(Level.SEVERE, "close() ros" + + DevUtils.extraInfo(), e); + } + } + + /** + * Return the internal RecordingInputStream + * + * @return A RIS. + */ + public RecordingInputStream getRecordedInput() { + return this.ris; + } + + /** + * @return The RecordingOutputStream. + */ + public RecordingOutputStream getRecordedOutput() { + return this.ros; + } + + /** + * Mark current position as the point where the HTTP headers end. + */ + public void markContentBegin() { + this.ris.markContentBegin(); + } + + public long getResponseContentLength() { + return this.ris.getResponseContentLength(); + } + + /** + * Close both input and output recorders. + * + * Recorders are the output streams to which we are recording. + * {@link #close()} closes the stream that is being recorded and the + * recorder. This method explicitly closes the recorder only. + */ + public void closeRecorders() { + try { + this.ris.closeRecorder(); + this.ros.closeRecorder(); + } catch (IOException e) { + DevUtils.warnHandle(e, "Convert to runtime exception?"); + } + } + + /** + * Cleanup backing files. + * + * Call when completely done w/ recorder. Removes any backing files that + * may have been dropped. + */ + public void cleanup() { + this.close(); + this.delete(this.backingFileBasename + RECORDING_OUTPUT_STREAM_SUFFIX); + this.delete(this.backingFileBasename + RECORDING_INPUT_STREAM_SUFFIX); + } + + /** + * Delete file if exists. + * + * @param name Filename to delete. + */ + private void delete(String name) { + File f = new File(name); + if (f.exists()) { + f.delete(); + } + } + + /** + * Get the current threads' HttpRecorder. + * + * @return This threads' HttpRecorder. Returns null if can't find a + * HttpRecorder in current instance. + */ + public static Recorder getHttpRecorder() { + Recorder recorder = null; + Thread thread = Thread.currentThread(); + if (thread instanceof RecorderMarker) { + recorder = ((RecorderMarker)thread).getHttpRecorder(); + } + return recorder; + } + + /** + * @param characterEncoding Character encoding of recording. + */ + public void setCharacterEncoding(String characterEncoding) { + this.characterEncoding = characterEncoding; + } + + /** + * @return Returns the characterEncoding. + */ + public String getCharacterEncoding() { + return this.characterEncoding; + } + + /** + * @return A ReplayCharSequence. Call close on the RCS when done w/ it. + * Will return indeterminate results if the underlying recording streams + * have not been closed first. + * @throws IOException + * @throws IOException + */ + public ReplayCharSequence getReplayCharSequence() throws IOException { + return getRecordedInput(). + getReplayCharSequence(this.characterEncoding); + } + + /** + * @return A replay input stream. + * @throws IOException + */ + public ReplayInputStream getReplayInputStream() throws IOException { + return getRecordedInput().getReplayInputStream(); + } + + /** + * Record the input stream for later playback by an extractor, etc. + * This is convenience method used to setup an artificial HttpRecorder + * scenario used in unit tests, etc. + * @param dir Directory to write backing file to. + * @param basename of what we're recording. + * @param in Stream to read. + * @param encoding Stream encoding. + * @throws IOException + * @return An {@link org.archive.util.Recorder}. + */ + public static Recorder wrapInputStreamWithHttpRecord(File dir, + String basename, InputStream in, String encoding) + throws IOException { + Recorder rec = new Recorder(dir, basename); + if (encoding != null && encoding.length() > 0) { + rec.setCharacterEncoding(encoding); + } + // Do not use FastBufferedInputStream here. It does not + // support mark. + InputStream is = rec.inputWrap(new BufferedInputStream(in)); + final int BUFFER_SIZE = 1024 * 4; + byte [] buffer = new byte[BUFFER_SIZE]; + while(true) { + // Just read it all down. + int x = is.read(buffer); + if (x == -1) { + break; + } + } + is.close(); + return rec; + } +} diff --git a/commons/src/main/java/org/archive/util/RecorderMarker.java b/commons/src/main/java/org/archive/util/RecorderMarker.java new file mode 100644 index 00000000..4bbf3bb7 --- /dev/null +++ b/commons/src/main/java/org/archive/util/RecorderMarker.java @@ -0,0 +1,38 @@ +/* HeritrixSSLProtocolSocketFactory + * + * Created on Feb 24, 2004 + * + * Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.util; + + +/** + * A marker interface to denote a class with a gettable HttpRecorder. + * + * @author stack + * @version $Id$ + */ +public interface RecorderMarker +{ + /** + * @return An instance of HttpRecorder. + */ + public Recorder getHttpRecorder(); +} diff --git a/commons/src/main/java/org/archive/util/Reporter.java b/commons/src/main/java/org/archive/util/Reporter.java new file mode 100644 index 00000000..27ea4aad --- /dev/null +++ b/commons/src/main/java/org/archive/util/Reporter.java @@ -0,0 +1,47 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.util; + +import java.io.IOException; +import java.io.PrintWriter; + +public interface Reporter { + /** + * Make a default report to the passed-in Writer. Should + * be equivalent to reportTo(null, writer) + * + * @param writer to receive report + */ + public void reportTo(PrintWriter writer) throws IOException; + + /** + * Write a short single-line summary report + * + * @param writer to receive report + */ + public void singleLineReportTo(PrintWriter pw) throws IOException; + + /** + * Return a legend for the single-line summary report as a String. + * + * @return String single-line summary legend + */ + public String singleLineLegend(); +} diff --git a/commons/src/main/java/org/archive/util/SURT.java b/commons/src/main/java/org/archive/util/SURT.java new file mode 100644 index 00000000..0e177d59 --- /dev/null +++ b/commons/src/main/java/org/archive/util/SURT.java @@ -0,0 +1,262 @@ +/* SURT +* +* $Id$ +* +* Created on Jul 16, 2004 +* +* Copyright (C) 2004 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.PrintStream; +import java.util.regex.Matcher; + +import org.apache.commons.httpclient.URIException; +import org.archive.net.UURIFactory; + +/** + * Sort-friendly URI Reordering Transform. + * + * Converts URIs of the form: + * + * scheme://userinfo@domain.tld:port/path?query#fragment + * + * ...into... + * + * scheme://(tld,domain,:port@userinfo)/path?query#fragment + * + * The '(' ')' characters serve as an unambiguous notice that the so-called + * 'authority' portion of the URI ([userinfo@]host[:port] in http URIs) has + * been transformed; the commas prevent confusion with regular hostnames. + * + * This remedies the 'problem' with standard URIs that the host portion of a + * regular URI, with its dotted-domains, is actually in reverse order from + * the natural hierarchy that's usually helpful for grouping and sorting. + * + * The value of respecting URI case variance is considered negligible: it + * is vanishingly rare for case-variance to be meaningful, while URI case- + * variance often arises from people's confusion or sloppiness, and they + * only correct it insofar as necessary to avoid blatant problems. Thus + * the usual SURT form is considered to be flattened to all lowercase, and + * not completely reversible. + * + * @author gojomo + */ +public class SURT { + static char DOT = '.'; + static String BEGIN_TRANSFORMED_AUTHORITY = "("; + static String TRANSFORMED_HOST_DELIM = ","; + static String END_TRANSFORMED_AUTHORITY = ")"; + + // 1: scheme:// + // 2: userinfo (if present) + // 3: @ (if present) + // 4: dotted-quad host + // 5: other host + // 6: :port + // 7: path + static String URI_SPLITTER = + "^(\\w+://)(?:([-\\w\\.!~\\*'\\(\\)%;:&=+$,]+?)(@))?"+ + // 1 2 3 + "(?:((?:\\d{1,3}\\.){3}\\d{1,3})|(\\S+?))(:\\d+)?(/\\S*)?$"; + // 4 5 6 7 + + // RFC2396 + // reserved = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" | + // "$" | "," + // unreserved = alphanum | mark + // mark = "-" | "_" | "." | "!" | "~" | "*" | "'" | "(" | ")" + // userinfo = *( unreserved | escaped | + // ";" | ":" | "&" | "=" | "+" | "$" | "," ) + // escaped = "%" hex hex + + + /** + * Utility method for creating the SURT form of the URI in the + * given String. + * + * By default, does not preserve casing. + * + * @param s String URI to be converted to SURT form + * @return SURT form + */ + public static String fromURI(String s) { + return fromURI(s,false); + } + + /** + * Utility method for creating the SURT form of the URI in the + * given String. + * + * If it appears a bit convoluted in its approach, note that it was + * optimized to minimize object-creation after allocation-sites profiling + * indicated this method was a top source of garbage in long-running crawls. + * + * Assumes that the String URI has already been cleaned/fixed (eg + * by UURI fixup) in ways that put it in its crawlable form for + * evaluation. + * + * @param s String URI to be converted to SURT form + * @param preserveCase whether original case should be preserved + * @return SURT form + */ + public static String fromURI(String s, boolean preserveCase) { + Matcher m = TextUtils.getMatcher(URI_SPLITTER,s); + if(!m.matches()) { + // not an authority-based URI scheme; return unchanged + TextUtils.recycleMatcher(m); + return s; + } + // preallocate enough space for SURT form, which includes + // 3 extra characters ('(', ')', and one more ',' than '.'s + // in original) + StringBuffer builder = new StringBuffer(s.length()+3); + append(builder,s,m.start(1),m.end(1)); // scheme:// + builder.append(BEGIN_TRANSFORMED_AUTHORITY); // '(' + + if(m.start(4)>-1) { + // dotted-quad ip match: don't reverse + append(builder,s,m.start(4),m.end(4)); + } else { + // other hostname match: do reverse + int hostSegEnd = m.end(5); + int hostStart = m.start(5); + for(int i = m.end(5)-1; i>=hostStart; i--) { + if(s.charAt(i-1)!=DOT && i > hostStart) { + continue; + } + append(builder,s,i,hostSegEnd); // rev host segment + builder.append(TRANSFORMED_HOST_DELIM); // ',' + hostSegEnd = i-1; + } + } + + append(builder,s,m.start(6),m.end(6)); // :port + append(builder,s,m.start(3),m.end(3)); // at + append(builder,s,m.start(2),m.end(2)); // userinfo + builder.append(END_TRANSFORMED_AUTHORITY); // ')' + append(builder,s,m.start(7),m.end(7)); // path + if (!preserveCase) { + for(int i = 0; i < builder.length(); i++) { + builder.setCharAt(i,Character.toLowerCase(builder.charAt((i)))); + } + } + TextUtils.recycleMatcher(m); + return builder.toString(); + } + + private static void append(StringBuffer b, CharSequence cs, int start, + int end) { + if (start < 0) { + return; + } + b.append(cs, start, end); + } + + /** + * Given a plain URI or hostname/hostname+path, deduce an implied SURT + * prefix from it. Results may be unpredictable on strings that cannot + * be interpreted as URIs. + * + * UURI 'fixup' is applied to the URI that is built. + * + * @param u URI or almost-URI to consider + * @return implied SURT prefix form + */ + public static String prefixFromPlain(String u) { + u = fromPlain(u); + // truncate to implied prefix + u = SurtPrefixSet.asPrefix(u); + return u; + } + + /** + * Given a plain URI or hostname/hostname+path, give its SURT form. + * Results may be unpredictable on strings that cannot + * be interpreted as URIs. + * + * UURI 'fixup' is applied to the URI before conversion to SURT + * form. + * + * @param u URI or almost-URI to consider + * @return implied SURT prefix form + */ + public static String fromPlain(String u) { + u = ArchiveUtils.addImpliedHttpIfNecessary(u); + boolean trailingSlash = u.endsWith("/"); + // ensure all typical UURI cleanup (incl. IDN-punycoding) is done + try { + u = UURIFactory.getInstance(u).toString(); + } catch (URIException e) { + e.printStackTrace(); + // allow to continue with original string uri + } + // except: don't let UURI-fixup add a trailing slash + // if it wasn't already there (presence or absence of + // such slash has special meaning specifying implied + // SURT prefixes) + if(!trailingSlash && u.endsWith("/")) { + u = u.substring(0,u.length()-1); + } + // convert to full SURT + u = SURT.fromURI(u); + return u; + } + + /** + * Allow class to be used as a command-line tool for converting + * URL lists (or naked host or host/path fragments implied + * to be HTTP URLs) to SURT form. Lines that cannot be converted + * are returned unchanged. + * + * + * Read from stdin or first file argument. Writes to stdout or + * second argument filename + * + * @param args cmd-line arguments + * @throws IOException + */ + public static void main(String[] args) throws IOException { + InputStream in = args.length > 0 ? new BufferedInputStream( + new FileInputStream(args[0])) : System.in; + PrintStream out = args.length > 1 ? new PrintStream( + new BufferedOutputStream(new FileOutputStream(args[1]))) + : System.out; + BufferedReader br = + new BufferedReader(new InputStreamReader(in)); + String line; + while((line = br.readLine())!=null) { + if(line.indexOf("#")>0) line=line.substring(0,line.indexOf("#")); + line = line.trim(); + if(line.length()==0) continue; + line = ArchiveUtils.addImpliedHttpIfNecessary(line); + out.println(SURT.fromURI(line)); + } + br.close(); + out.close(); + } +} diff --git a/commons/src/main/java/org/archive/util/SubList.java b/commons/src/main/java/org/archive/util/SubList.java new file mode 100644 index 00000000..4a1fc36d --- /dev/null +++ b/commons/src/main/java/org/archive/util/SubList.java @@ -0,0 +1,150 @@ +/* SubList + * + * $Id$ + * Created on September 23, 2006 + * + * Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.util; + + +import java.io.Serializable; +import java.util.AbstractList; +import java.util.List; + + +/** + * Universal sublist implementation. Instances of this class are + * appropriate to return from {@link List#subList(int, int)} + * implementations. + * + *

This implementation is efficient if the super list is random-access. + * LinkedList-style super lists should subclass this and provide a custom + * iterator. + * + * @author pjack + * @param the element type of the list + */ +public class SubList extends AbstractList implements Serializable { + + private static final long serialVersionUID = 1L; + + /** + * The list that created this SubList. + */ + final private List delegate; + + /** + * The starting index of the SubList, inclusive. + */ + private int start; + + /** + * The ending index of the SubList, exclusive. + */ + private int end; + + + /** + * Constructor. + * + * @param delegate the list that create this SubList + * @param start the starting index of the sublist, inclusive + * @param end the ending index of the sublist, exclusive + * @throws IndexOutOfBoundsException if start or end are outside the + * bounds of the list + * @throws IllegalArgumentException if end is less than start + */ + public SubList(List delegate, int start, int end) { + if ((start < 0) || (start > delegate.size())) { + throw new IndexOutOfBoundsException(); + } + if ((end < 0) || (end > delegate.size())) { + throw new IndexOutOfBoundsException(); + } + if (end < start) { + throw new IllegalArgumentException(); + } + this.delegate = delegate; + this.start = start; + this.end = end; + } + + /** + * Ensures that the given index is strictly within the bounds of this + * SubList. + * + * @param index the index to check + * @throws IndexOutOfBoundsException if the index is out of bounds + */ + private void ensureInside(int index) { + if ((index < 0) || (index >= end - start)) { + throw new IndexOutOfBoundsException(); + } + } + + /** + * Ensures that the given index is either within bounds or on the border + * of this SubList. In other words, this method allows the given index + * to be equal to {@link #size()}. + * + * @param index the index to check + * @throws IndexOutOfBoundsException if the index is out of bounds + */ + private void ensureBorder(int index) { + if ((index < 0) || (index > end - start)) { + throw new IndexOutOfBoundsException(); + } + } + + + @Override + public E get(int index) { + ensureInside(index); + return delegate.get(start + index); + } + + + @Override + public int size() { + return end - start; + } + + + @Override + public E set(int index, E value) { + ensureInside(index); + return delegate.set(start + index, value); + } + + + @Override + public void add(int index, E value) { + ensureBorder(index); + delegate.add(start + index, value); + end++; + } + + + @Override + public E remove(int index) { + ensureInside(index); + return delegate.remove(start + index); + } +} diff --git a/commons/src/main/java/org/archive/util/SurtPrefixSet.java b/commons/src/main/java/org/archive/util/SurtPrefixSet.java new file mode 100644 index 00000000..e5b8b80d --- /dev/null +++ b/commons/src/main/java/org/archive/util/SurtPrefixSet.java @@ -0,0 +1,344 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.util; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.PrintStream; +import java.io.Reader; +import java.util.Iterator; + +import org.archive.net.UURI; +import org.archive.util.iterator.LineReadingIterator; +import org.archive.util.iterator.RegexpLineIterator; + +/** + * Specialized TreeSet for keeping a set of String prefixes. + * + * Redundant prefixes (those that are themselves prefixed + * by other set entries) are eliminated. + * + * @author gojomo + */ +public class SurtPrefixSet extends PrefixSet { + + private static final long serialVersionUID = 2598365040524933110L; + + private static final String SURT_PREFIX_DIRECTIVE = "+"; + + + /** + * Read a set of SURT prefixes from a reader source; keep sorted and + * with redundant entries removed. + * + * @param r reader over file of SURT_format strings + * @throws IOException + */ + public void importFrom(Reader r) { + BufferedReader reader = new BufferedReader(r); + String s; + + Iterator iter = + new RegexpLineIterator( + new LineReadingIterator(reader), + RegexpLineIterator.COMMENT_LINE, + RegexpLineIterator.NONWHITESPACE_ENTRY_TRAILING_COMMENT, + RegexpLineIterator.ENTRY); + + while (iter.hasNext()) { + s = (String) iter.next(); + add(s.toLowerCase()); + } + } + + /** + * @param r Where to read from. + */ + public void importFromUris(Reader r) { + BufferedReader reader = new BufferedReader(r); + String s; + + Iterator iter = + new RegexpLineIterator( + new LineReadingIterator(reader), + RegexpLineIterator.COMMENT_LINE, + RegexpLineIterator.NONWHITESPACE_ENTRY_TRAILING_COMMENT, + RegexpLineIterator.ENTRY); + + while (iter.hasNext()) { + s = (String) iter.next(); + // s is a URI (or even fragmentary hostname), not a SURT + addFromPlain(s); + } + } + + /** + * Import SURT prefixes from a reader with mixed URI and SURT prefix + * format. + * + * @param r the reader to import the prefixes from + * @param deduceFromSeeds true to also import SURT prefixes implied + * from normal URIs/hostname seeds + */ + public void importFromMixed(Reader r, boolean deduceFromSeeds) { + BufferedReader reader = new BufferedReader(r); + String s; + + Iterator iter = + new RegexpLineIterator( + new LineReadingIterator(reader), + RegexpLineIterator.COMMENT_LINE, + RegexpLineIterator.NONWHITESPACE_ENTRY_TRAILING_COMMENT, + RegexpLineIterator.ENTRY); + + while (iter.hasNext()) { + s = (String) iter.next(); + if(s.startsWith(SURT_PREFIX_DIRECTIVE)) { + // it's specifically a SURT prefix line + String u = s.substring(SURT_PREFIX_DIRECTIVE.length()).trim(); + if(u.indexOf("(")>0) { + // formal SURT prefix; toLowerCase just in case + add(u.toLowerCase()); + } else { + // hostname/normal form URI from which + // to deduce SURT prefix + addFromPlain(u); + } + + continue; + } else { + if(deduceFromSeeds) { + // also deducing 'implied' SURT prefixes + // from normal URIs/hostname seeds + addFromPlain(s); + } + } + } + } + + /** + * Given a plain URI or hostname, deduce an implied SURT prefix from + * it and add to active prefixes. + * + * @param u String of URI or hostname + */ + public void addFromPlain(String u) { + u = prefixFromPlainForceHttp(u); + add(u); + } + + /** + * Given a plain URI or hostname/hostname+path, deduce an implied SURT + * prefix from it. Results may be unpredictable on strings that cannot + * be interpreted as URIs. + * + * UURI 'fixup' is applied to the URI that is built. + * + * HTTPS URIs are changed to HTTP as a convenience for the usual + * preferred common-treatment. + * + * @param u URI or almost-URI to consider + * @return implied SURT prefix form + */ + public static String prefixFromPlainForceHttp(String u) { + u = SURT.prefixFromPlain(u); + u = coerceFromHttpsForComparison(u); + return u; + } + + /** + * For SURT comparisons -- prefixes or candidates being checked against + * those prefixes -- we treat https URIs as if they were http. + * + * @param u string to coerce if it has https scheme + * @return string converted to http scheme, or original if not necessary + */ + private static String coerceFromHttpsForComparison(String u) { + if (u.startsWith("https://")) { + u = "http" + u.substring("https".length()); + } + return u; + } + + /** + * Utility method for truncating a SURT that came from a + * full URI (as a seed, for example) into a prefix + * for determining inclusion. + * + * This involves: + * 

+     *    (1) removing the last path component, if any
+     *        (anything after the last '/', if there are
+     *        at least 3 '/'s)
+     *    (2) removing a trailing ')', if present, opening
+     *        the possibility of proper subdomains. (This
+     *        means that the presence or absence of a
+     *        trailing '/' after a hostname in a seed list
+     *        is significant for the how the SURT prefix is 
+     *        created, even though it is not signficant for 
+     *        the URI's treatment as a seed.)
+     *
+ * + * @param s String to work on. + * @return As prefix. + */ + public static String asPrefix(String s) { + // Strip last path-segment, if more than 3 slashes + s = s.replaceAll("^(.*//.*/)[^/]*","$1"); + // Strip trailing ")", if present and NO path (no 3rd slash). + if (!s.endsWith("/")) { + s = s.replaceAll("^(.*)\\)","$1"); + } + return s; + } + + /** + * Calculate the SURT form URI to use as a candidate against prefixes + * from the given Object (CandidateURI or UURI) + * + * @param object CandidateURI or UURI + * @return SURT form of URI for evaluation, or null if unavailable + */ + public static String getCandidateSurt(UURI u) { + if (u == null) { + return null; + } + String candidateSurt = u.getSurtForm(); + // also want to treat https as http + candidateSurt = coerceFromHttpsForComparison(candidateSurt); + return candidateSurt; + } + /** + * @param fw + * @throws IOException + */ + public void exportTo(FileWriter fw) throws IOException { + Iterator iter = this.iterator(); + while(iter.hasNext()) { + fw.write((String)iter.next() + "\n"); + } + } + + /** + * Changes all prefixes so that they enforce an exact host. For + * prefixes that already include a ')', this means discarding + * anything after ')' (path info). For prefixes that don't include + * a ')' -- domain prefixes open to subdomains -- add the closing + * ')' (or ",)"). + */ + public void convertAllPrefixesToHosts() { + SurtPrefixSet iterCopy = (SurtPrefixSet) this.clone(); + Iterator iter = iterCopy.iterator(); + while (iter.hasNext()) { + String prefix = (String) iter.next(); + String convPrefix = convertPrefixToHost(prefix); + if(prefix!=convPrefix) { + // if returned value not unchanged, update set + this.remove(prefix); + this.add(convPrefix); + } + } + } + + public static String convertPrefixToHost(String prefix) { + if(prefix.endsWith(")")) { + return prefix; // no change necessary + } + if(prefix.indexOf(')')<0) { + // open-ended domain prefix + if(!prefix.endsWith(",")) { + prefix += ","; + } + prefix += ")"; + } else { + // prefix with excess path-info + prefix = prefix.substring(0,prefix.indexOf(')')+1); + } + return prefix; + } + + /** + * Changes all prefixes so that they only enforce a general + * domain (allowing subdomains).For prefixes that don't include + * a ')', no change is necessary. For others, truncate everything + * from the ')' onward. Additionally, truncate off "www," if it + * appears. + */ + public void convertAllPrefixesToDomains() { + SurtPrefixSet iterCopy = (SurtPrefixSet) this.clone(); + Iterator iter = iterCopy.iterator(); + while (iter.hasNext()) { + String prefix = (String) iter.next(); + String convPrefix = convertPrefixToDomain(prefix); + if(prefix!=convPrefix) { + // if returned value not unchanged, update set + this.remove(prefix); + this.add(convPrefix); + } + } + } + + public static String convertPrefixToDomain(String prefix) { + if(prefix.indexOf(')')>=0) { + prefix = prefix.substring(0,prefix.indexOf(')')); + } + // strip 'www,' when present + if(prefix.endsWith("www,")) { + prefix = prefix.substring(0,prefix.length()-4); + } + return prefix; + } + + /** + * Allow class to be used as a command-line tool for converting + * URL lists (or naked host or host/path fragments implied + * to be HTTP URLs) to implied SURT prefix form. + * + * Read from stdin or first file argument. Writes to stdout. + * + * @param args cmd-line arguments: may include input file + * @throws IOException + */ + public static void main(String[] args) throws IOException { + InputStream in = args.length > 0 ? new BufferedInputStream( + new FileInputStream(args[0])) : System.in; + PrintStream out = args.length > 1 ? new PrintStream( + new BufferedOutputStream(new FileOutputStream(args[1]))) + : System.out; + BufferedReader br = + new BufferedReader(new InputStreamReader(in)); + String line; + while((line = br.readLine())!=null) { + if(line.indexOf("#")>0) line=line.substring(0,line.indexOf("#")); + line = line.trim(); + if(line.length()==0) continue; + out.println(prefixFromPlainForceHttp(line)); + } + br.close(); + out.close(); + } +} diff --git a/commons/src/main/java/org/archive/util/TestUtils.java b/commons/src/main/java/org/archive/util/TestUtils.java new file mode 100644 index 00000000..a408444d --- /dev/null +++ b/commons/src/main/java/org/archive/util/TestUtils.java @@ -0,0 +1,131 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.util; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.lang.ref.SoftReference; +import java.util.Arrays; +import java.util.LinkedList; + +import junit.framework.TestCase; +import junit.framework.TestSuite; + + +/** + * Utility methods useful in testing situations. + * + * @author gojomo + */ +public class TestUtils { + + /** + * Temporarily exhaust memory, forcing weak/soft references to + * be broken. + */ + public static void forceScarceMemory() { + // force soft references to be broken + LinkedList> hog = new LinkedList>(); + long blocks = Runtime.getRuntime().maxMemory() / 1000000; + for(long l = 0; l <= blocks; l++) { + try { + hog.add(new SoftReference(new byte[1000000])); + } catch (OutOfMemoryError e) { + hog = null; + break; + } + } + } + + + public static void testSerialization(Object proc) throws Exception { + byte[] first = serialize(proc); + ByteArrayInputStream binp = new ByteArrayInputStream(first); + ObjectInputStream oinp = new ObjectInputStream(binp); + Object o = oinp.readObject(); + oinp.close(); + TestCase.assertEquals(proc.getClass(), o.getClass()); + byte[] second = serialize(o); + TestCase.assertTrue(Arrays.equals(first, second)); + } + + + private static byte[] serialize(Object o) throws Exception { + ByteArrayOutputStream bout = new ByteArrayOutputStream(); + ObjectOutputStream oout = new ObjectOutputStream(bout); + oout.writeObject(o); + oout.close(); + return bout.toByteArray(); + } + + + public static TestSuite makePackageSuite(Class c) + throws ClassNotFoundException { + String cname = c.getName(); + int p = cname.lastIndexOf('.'); + String dir = cname.substring(0, p).replace('.', File.separatorChar); + String root = "heritrix/src/test/java/".replace('/', File.separatorChar); + File src = new File(root); + return makeSuite(src, new File(root + dir)); + } + + + public static TestSuite makeSuite(File srcRoot, File dir) + throws ClassNotFoundException { + TestSuite result = new TestSuite("All Tests"); + if (!dir.exists()) { + throw new IllegalArgumentException(dir + " does not exist."); + } + scanSuite(result, srcRoot, dir); + return result; + } + + + private static void scanSuite(TestSuite suite, File start, File dir) + throws ClassNotFoundException { + for (File f: dir.listFiles()) { + if (f.isDirectory() && !f.getName().startsWith(".")) { + String prefix = start.getAbsolutePath(); + String full = f.getAbsolutePath(); + TestSuite sub = new TestSuite(full.substring(prefix.length())); + scanSuite(sub, start, f); + if (sub.testCount() > 0) { + suite.addTest(sub); + } + } else { + if (f.getName().endsWith("Test.java")) { + String full = f.getAbsolutePath(); + String prefix = start.getAbsolutePath(); + String cname = full.substring(prefix.length()); + if (cname.startsWith(File.separator)) { + cname = cname.substring(1); + } + cname = cname.replace(File.separatorChar, '.'); + cname = cname.substring(0, cname.length() - 5); + suite.addTestSuite(Class.forName(cname)); + } + } + } + } + +} diff --git a/commons/src/main/java/org/archive/util/TextUtils.java b/commons/src/main/java/org/archive/util/TextUtils.java new file mode 100644 index 00000000..94fdfe86 --- /dev/null +++ b/commons/src/main/java/org/archive/util/TextUtils.java @@ -0,0 +1,254 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.util; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringReader; +import java.io.StringWriter; +import java.io.Writer; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.lang.StringEscapeUtils; + +public class TextUtils { + private static final String FIRSTWORD = "^([^\\s]*).*$"; + + private static final ThreadLocal> TL_MATCHER_MAP + = new ThreadLocal>() { + protected Map initialValue() { + return new HashMap(50); + } + }; + + /** + * Get a matcher object for a precompiled regex pattern. + * + * This method tries to reuse Matcher objects for efficiency. + * It can hold for recycling one Matcher per pattern per thread. + * + * Matchers retrieved should be returned for reuse via the + * recycleMatcher() method, but no errors will occur if they + * are not. + * + * This method is a hotspot frequently accessed. + * + * @param pattern the string pattern to use + * @param input the character sequence the matcher should be using + * @return a matcher object loaded with the submitted character sequence + */ + public static Matcher getMatcher(String pattern, CharSequence input) { + if (pattern == null) { + throw new IllegalArgumentException("String 'pattern' must not be null"); + } + input = new InterruptibleCharSequence(input); + final Map matchers = TL_MATCHER_MAP.get(); + Matcher m = (Matcher)matchers.get(pattern); + if(m == null) { + m = Pattern.compile(pattern).matcher(input); + } else { + matchers.put(pattern,null); + m.reset(input); + } + return m; + } + + public static void recycleMatcher(Matcher m) { + final Map matchers = TL_MATCHER_MAP.get(); + matchers.put(m.pattern().pattern(),m); + } + + /** + * Utility method using a precompiled pattern instead of using the + * replaceAll method of the String class. This method will also be reusing + * Matcher objects. + * + * @see java.util.regex.Pattern + * @param pattern precompiled Pattern to match against + * @param input the character sequence to check + * @param replacement the String to substitute every match with + * @return the String with all the matches substituted + */ + public static String replaceAll( + String pattern, CharSequence input, String replacement) { + input = new InterruptibleCharSequence(input); + Matcher m = getMatcher(pattern, input); + String res = m.replaceAll(replacement); + recycleMatcher(m); + return res; + } + + /** + * Utility method using a precompiled pattern instead of using the + * replaceFirst method of the String class. This method will also be reusing + * Matcher objects. + * + * @see java.util.regex.Pattern + * @param pattern precompiled Pattern to match against + * @param input the character sequence to check + * @param replacement the String to substitute the first match with + * @return the String with the first match substituted + */ + public static String replaceFirst( + String pattern, CharSequence input, String replacement) { + input = new InterruptibleCharSequence(input); + Matcher m = getMatcher(pattern, input); + String res = m.replaceFirst(replacement); + recycleMatcher(m); + return res; + } + + /** + * Utility method using a precompiled pattern instead of using the matches + * method of the String class. This method will also be reusing Matcher + * objects. + * + * @see java.util.regex.Pattern + * @param pattern precompiled Pattern to match against + * @param input the character sequence to check + * @return true if character sequence matches + */ + public static boolean matches(String pattern, CharSequence input) { + input = new InterruptibleCharSequence(input); + Matcher m = getMatcher(pattern, input); + boolean res = m.matches(); + recycleMatcher(m); + return res; + } + + /** + * Utility method using a precompiled pattern instead of using the split + * method of the String class. + * + * @see java.util.regex.Pattern + * @param pattern precompiled Pattern to split by + * @param input the character sequence to split + * @return array of Strings split by pattern + */ + public static String[] split(String pattern, CharSequence input) { + input = new InterruptibleCharSequence(input); + Matcher m = getMatcher(pattern,input); + String[] retVal = m.pattern().split(input); + recycleMatcher(m); + return retVal; + } + + /** + * @param s String to find first word in (Words are delimited by + * whitespace). + * @return First word in the passed string else null if no word found. + */ + public static String getFirstWord(String s) { + Matcher m = getMatcher(FIRSTWORD, s); + String retVal = (m != null && m.matches())? m.group(1): null; + recycleMatcher(m); + return retVal; + } + + /** + * Escapes a string so that it can be passed as an argument to a javscript + * in a JSP page. This method takes a string and returns the same string + * with any single quote escaped by prepending the character with a + * backslash. Linebreaks are also replaced with '\n'. Also, + * less-than signs and ampersands are replaced with HTML entities. + * + * @param s The string to escape + * @return The same string escaped. + */ + public static String escapeForHTMLJavascript(String s) { + return escapeForHTML(StringEscapeUtils.escapeJavaScript(s)); + } + + /** + * Escapes a string so that it can be placed inside XML/HTML attribute. + * Replaces ampersand, less-than, greater-than, single-quote, and + * double-quote with escaped versions. + * @param s The string to escape + * @return The same string escaped. + */ + public static String escapeForMarkupAttribute(String s) { + return StringEscapeUtils.escapeXml(s); + } + + /** + * Minimally escapes a string so that it can be placed inside XML/HTML + * attribute. + * Escapes lt and amp. + * @param s The string to escape + * @return The same string escaped. + */ + public static String escapeForHTML(String s) { + // TODO: do this in a single pass instead of creating 5 junk strings + String escaped = s.replaceAll("&","&"); + return escaped.replaceAll("<","<"); + } + + /** + * Utility method for writing a (potentially large) String to a JspWriter, + * escaping it for HTML display, without constructing another large String + * of the whole content. + * @param s String to write + * @param out destination JspWriter + * @throws IOException + */ + public static void writeEscapedForHTML(String s, Writer w) + throws IOException { + PrintWriter out = new PrintWriter(w); + BufferedReader reader = new BufferedReader(new StringReader(s)); + String line; + while((line=reader.readLine()) != null){ + out.println(StringEscapeUtils.escapeHtml(line)); + } + } + + /** + * Replaces HTML Entity Encodings. + * @param cs The CharSequence to remove html codes from + * @return the same CharSequence or an escaped String. + */ + public static CharSequence unescapeHtml(final CharSequence cs) { + if (cs == null) { + return cs; + } + + return StringEscapeUtils.unescapeHtml(cs.toString()); + } + + /** + * @param message Message to put at top of the string returned. May be + * null. + * @param e Exception to write into a string. + * @return Return formatted string made of passed message and stack trace + * of passed exception. + */ + public static String exceptionToString(String message, Throwable e) { + StringWriter sw = new StringWriter(); + if (message == null || message.length() == 0) { + sw.write(message); + sw.write("\n"); + } + e.printStackTrace(new PrintWriter(sw)); + return sw.toString(); + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/util/TimestampSerialno.java b/commons/src/main/java/org/archive/util/TimestampSerialno.java new file mode 100644 index 00000000..4a46c9a3 --- /dev/null +++ b/commons/src/main/java/org/archive/util/TimestampSerialno.java @@ -0,0 +1,62 @@ +/* TimestampSerialno +* +* $Id$ +* +* Created July 19th, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util; + +/** + * Immutable data structure that holds a timestamp and an accompanying + * serial number. + * + * For Igor! + * + * @author stack + */ +public class TimestampSerialno { + private final String ts; + private final int serialNumber; + + public TimestampSerialno(String ts, int serialNo) { + this.ts = ts; + this.serialNumber = serialNo; + } + + public TimestampSerialno(int serialNo) { + this.ts = ArchiveUtils.get14DigitDate(); + this.serialNumber = serialNo; + } + + /** + * @return Returns the now. + */ + public String getTimestamp() { + return this.ts; + } + + /** + * @return Returns the serialNumber. + */ + public int getSerialNumber() { + return this.serialNumber; + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/util/TmpDirTestCase.java b/commons/src/main/java/org/archive/util/TmpDirTestCase.java new file mode 100644 index 00000000..575a76b4 --- /dev/null +++ b/commons/src/main/java/org/archive/util/TmpDirTestCase.java @@ -0,0 +1,122 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.util; + +import java.io.File; +import java.io.IOException; + +import junit.framework.TestCase; + + +/** + * Base class for TestCases that want access to a tmp dir for the writing + * of files. + * + * @author stack + */ +public abstract class TmpDirTestCase extends TestCase +{ + /** + * Name of the system property that holds pointer to tmp directory into + * which we can safely write files. + */ + public static final String TEST_TMP_SYSTEM_PROPERTY_NAME = "testtmpdir"; + + /** + * Default test tmp. + */ + public static final String DEFAULT_TEST_TMP_DIR = File.separator + "tmp" + + File.separator + "heritrix-junit-tests"; + + /** + * Directory to write temporary files to. + */ + private File tmpDir = null; + + + public TmpDirTestCase() + { + super(); + } + + public TmpDirTestCase(String testName) + { + super(testName); + } + + /* + * @see TestCase#setUp() + */ + protected void setUp() throws Exception { + super.setUp(); + this.tmpDir = tmpDir(); + } + + /** + * @return Returns the tmpDir. + */ + public File getTmpDir() + { + return this.tmpDir; + } + + /** + * Delete any files left over from previous run. + * + * @param basename Base name of files we're to clean up. + */ + public void cleanUpOldFiles(String basename) { + cleanUpOldFiles(getTmpDir(), basename); + } + + /** + * Delete any files left over from previous run. + * + * @param prefix Base name of files we're to clean up. + * @param basedir Directory to start cleaning in. + */ + public void cleanUpOldFiles(File basedir, String prefix) { + File [] files = FileUtils.getFilesWithPrefix(basedir, prefix); + if (files != null) { + for (int i = 0; i < files.length; i++) { + org.apache.commons.io.FileUtils.deleteQuietly(files[i]); + } + } + } + + + public static File tmpDir() throws IOException { + String tmpDirStr = System.getProperty(TEST_TMP_SYSTEM_PROPERTY_NAME); + tmpDirStr = (tmpDirStr == null)? DEFAULT_TEST_TMP_DIR: tmpDirStr; + File tmpDir = new File(tmpDirStr); + if (!tmpDir.exists()) + { + tmpDir.mkdirs(); + } + + if (!tmpDir.canWrite()) + { + throw new IOException(tmpDir.getAbsolutePath() + + " is unwriteable."); + } + + return tmpDir; + } +} diff --git a/commons/src/main/java/org/archive/util/Transform.java b/commons/src/main/java/org/archive/util/Transform.java new file mode 100644 index 00000000..99f8c91c --- /dev/null +++ b/commons/src/main/java/org/archive/util/Transform.java @@ -0,0 +1,160 @@ +/* Transform +* +* $Id$ +* +* Created on September 26, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util; + +import java.util.AbstractCollection; +import java.util.Collection; +import java.util.Iterator; + + +/** + * A transformation of a collection. The elements in the transform are based + * on the elements of some other collection; the original collection's + * elements are transformed using a specified transformer. Changes to the + * original collection are automatically reflected in the transform and + * vice-versa. + * + *

If the transformer returns null for a given original object, then that + * object will not be included in the transform. Thus the transform might + * be smaller than the original collection. Note that Transform instances + * can never contain the null element. + * + *

This collection implementation does not support the optional add + * operation. + * + * @author pjack + * + * @param the type of the original elements in the collection + * @param the type of the tranformed elements + */ +public class Transform +extends AbstractCollection { + + /** The original collection. */ + final private Collection delegate; + + /** Transforms the original objects. */ + final private Transformer transformer; + + /** + * Constructor. + * + * @param delegate The collection whose elements to transform. + * @param transformer Transforms the elements + */ + public Transform(Collection delegate, + Transformer transformer) { + this.delegate = delegate; + this.transformer = transformer; + } + + public int size() { + int count = 0; + Iterator iter = iterator(); + while (iter.hasNext()) { + iter.next(); + count++; + } + return count; + } + + public Iterator iterator() { + return new TransformIterator( + delegate.iterator(), transformer); + } + + + /** + * Returns a transform containing only objects of a given class. + * + * @param the target class + * @param c the collection to transform + * @param cls the class of objects to return + * @return a collection containing only objects of class cls + */ + public static Collection subclasses( + Collection c, + final Class cls) { + Transformer t = new Transformer() { + public Target transform(Object s) { + if (cls.isInstance(s)) { + return cls.cast(s); + } else { + return null; + } + } + }; + return new Transform(c, t); + } +} + + +class TransformIterator implements Iterator { + + final private Iterator iterator; + final private Transformer transformer; + private Transformed next; + + public TransformIterator(Iterator iterator, + Transformer transformer) { + this.iterator = iterator; + this.transformer = transformer; + } + + public boolean hasNext() { + if (next != null) { + return true; + } + while (iterator.hasNext()) { + Original o = iterator.next(); + next = transformer.transform(o); + if (next != null) { + return true; + } + } + return false; + } + + public Transformed next() { + if (!hasNext()) { + throw new IllegalStateException(); + } + Transformed r = next; + next = null; + return r; + } + + // FIXME: this can break standard Iterator contract, for example + // transformIterator.next(); + // if(transformIterator.hasNext()) { + // transformIterator.remove(); + // } + // usual iterator contract is to remove the last object returned + // by next; in this case the subsequent + public void remove() { + iterator.remove(); + } + +} diff --git a/commons/src/main/java/org/archive/util/Transformer.java b/commons/src/main/java/org/archive/util/Transformer.java new file mode 100644 index 00000000..54a2a37b --- /dev/null +++ b/commons/src/main/java/org/archive/util/Transformer.java @@ -0,0 +1,46 @@ +/* Transformer +* +* $Id$ +* +* Created on September 26, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util; + + +/** + * Transforms objects from one thing into another. + * + * @author pjack + * + * @param the type of the original objects + * @param the type of the transformed objects + */ +public interface Transformer { + + /** + * Transforms the given object. + * + * @param o the object to transform + * @return the transformed object + */ + Transformed transform(Original o); + +} diff --git a/commons/src/main/java/org/archive/util/anvl/ANVLRecord.java b/commons/src/main/java/org/archive/util/anvl/ANVLRecord.java new file mode 100644 index 00000000..4544a7bc --- /dev/null +++ b/commons/src/main/java/org/archive/util/anvl/ANVLRecord.java @@ -0,0 +1,333 @@ +/* ANVLRecord +* +* $Id$ +* +* Created on July 26, 2006. +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util.anvl; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.archive.io.UTF8Bytes; + +/** + * An ordered {@link List} with 'data' {@link Element} values. + * ANVLRecords end with a blank line. + * + * @see A Name-Value + * Language (ANVL) + * @author stack + */ +public class ANVLRecord extends ArrayList implements UTF8Bytes { + private static final long serialVersionUID = -4610638888453052958L; + + public static final String MIMETYPE = "application/warc-fields"; + + public static final ANVLRecord EMPTY_ANVL_RECORD = new ANVLRecord(); + + /** + * Arbitrary upper bound on maximum size of ANVL Record. + * Will throw an IOException if exceed this size. + */ + public static final long MAXIMUM_SIZE = 1024 * 10; + + /** + * An ANVL 'newline'. + * @see http://en.wikipedia.org/wiki/CRLF + */ + static final String CRLF = "\r\n"; + + static final String FOLD_PREFIX = CRLF + ' '; + + public ANVLRecord() { + super(); + } + + public ANVLRecord(Collection c) { + super(c); + } + + public ANVLRecord(int initialCapacity) { + super(initialCapacity); + } + + public boolean addLabel(final String l) { + return super.add(new Element(new Label(l))); + } + + public boolean addLabelValue(final String l, final String v) { + return super.add(new Element(new Label(l), new Value(v))); + } + + @Override + public String toString() { + // TODO: What to emit for empty ANVLRecord? + StringBuilder sb = new StringBuilder(); + for (final Iterator i = iterator(); i.hasNext();) { + sb.append(i.next()); + sb.append(CRLF); + } + // 'ANVL Records end in a blank line'. + sb.append(CRLF); + return sb.toString(); + } + + public Map asMap() { + Map m = new HashMap(size()); + for (final Iterator i = iterator(); i.hasNext();) { + Element e = i.next(); + m.put(e.getLabel().toString(), + e.isValue()? e.getValue().toString(): (String)null); + } + return m; + } + + @Override + public ANVLRecord clone() { + return new ANVLRecord(this); + } + + /** + * @return This ANVLRecord as UTF8 bytes. + */ + public byte [] getUTF8Bytes() + throws UnsupportedEncodingException { + return toString().getBytes(UTF8); + } + + /** + * Parses a single ANVLRecord from passed InputStream. + * Read as a single-byte stream until we get to a CRLFCRLF which + * signifies End-of-ANVLRecord. Then parse all read as a UTF-8 Stream. + * Doing it this way, while requiring a double-scan, it makes it so do not + * need to be passed a RepositionableStream or a Stream that supports + * marking. Also no danger of over-reading which can happen when we + * wrap passed Stream with an InputStreamReader for doing UTF-8 + * character conversion (See the ISR class comment). + * @param is InputStream + * @return An ANVLRecord instance. + * @throws IOException + */ + public static ANVLRecord load(final InputStream is) + throws IOException { + // It doesn't look like a CRLF sequence is possible in UTF-8 without + // it signifying CRLF: The top bits are set in multibyte characters. + // Was thinking of recording CRLF as I was running through this first + // parse but the offsets would then be incorrect if any multibyte + // characters in the intervening gaps between CRLF. + boolean isCRLF = false; + boolean recordStart = false; + ByteArrayOutputStream baos = new ByteArrayOutputStream(1024); + boolean done = false; + int read = 0; + for (int c = -1, previousCharacter; !done;) { + if (read++ >= MAXIMUM_SIZE) { + throw new IOException("Read " + MAXIMUM_SIZE + + " bytes without finding \\r\\n\\r\\n " + + "End-Of-ANVLRecord"); + } + previousCharacter = c; + c = is.read(); + if (c == -1) { + throw new IOException("End-Of-Stream before \\r\\n\\r\\n " + + "End-Of-ANVLRecord:\n" + + new String(baos.toByteArray(), UTF8)); + } + if (isLF((char)c) && isCR((char)previousCharacter)) { + if (isCRLF) { + // If we just had a CRLF, then its two CRLFs and its end of + // record. We're done. + done = true; + } else { + isCRLF = true; + } + } else if (!recordStart && Character.isWhitespace(c)) { + // Skip any whitespace at start of ANVLRecord. + continue; + } else { + // Clear isCRLF flag if this character is NOT a '\r'. + if (isCRLF && !isCR((char)c)) { + isCRLF = false; + } + // Not whitespace so start record if we haven't already. + if (!recordStart) { + recordStart = true; + } + } + baos.write(c); + } + return load(new String(baos.toByteArray(), UTF8)); + } + + /** + * Parse passed String for an ANVL Record. + * Looked at writing javacc grammer but preprocessing is required to + * handle folding: See + * https://javacc.dev.java.net/servlets/BrowseList?list=users&by=thread&from=56173. + * Looked at Terence Parr's ANTLR. More capable. Can set lookahead count. + * A value of 3 would help with folding. But its a pain defining UNICODE + * grammers -- needed by ANVL -- and support seems incomplete + * anyways: http://www.doc.ic.ac.uk/lab/secondyear/Antlr/lexer.html#unicode. + * For now, go with the below hand-rolled parser. + * @param s String with an ANVLRecord. + * @return ANVLRecord parsed from passed String. + * @throws IOException + */ + public static ANVLRecord load(final String s) + throws IOException { + ANVLRecord record = new ANVLRecord(); + boolean inValue = false, inLabel = false, inComment = false, + inNewLine = false; + String label = null; + StringBuilder sb = new StringBuilder(s.length()); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + + // Assert I can do look-ahead. + if ((i + 1) > s.length()) { + throw new IOException("Premature End-of-ANVLRecord:\n" + + s.substring(i)); + } + + // If at LF of a CRLF, just go around again. Eat up the LF. + if (inNewLine && isLF(c)) { + continue; + } + + // If we're at a CRLF and we were just on one, exit. Found Record. + if (inNewLine && isCR(c) && isLF(s.charAt(i + 1))) { + break; + } + + // Check if we're on a fold inside a Value. Skip multiple white + // space after CRLF. + if (inNewLine && inValue && Character.isWhitespace(c)) { + continue; + } + + // Else set flag if we're at a CRLF. + inNewLine = isCR(c) && isLF(s.charAt(i + 1)); + + if (inNewLine) { + if (inComment) { + inComment = false; + } else if (label != null && !inValue) { + // Label only 'data element'. + record.addLabel(label); + label = null; + sb.setLength(0); + } else if (inValue) { + // Assert I can do look-ahead past current CRLF. + if ((i + 3) > s.length()) { + throw new IOException("Premature End-of-ANVLRecord " + + "(2):\n" + s.substring(i)); + } + if (!isCR(s.charAt(i + 2)) && !isLF(s.charAt(i + 3)) + && Character.isWhitespace(s.charAt(i + 2))) { + // Its a fold. Let it go around. But add in a CRLF and + // space and do it here. We don't let CRLF fall through + // to the sb.append on the end of this loop. + sb.append(CRLF); + sb.append(' '); + } else { + // Next line is a new SubElement, a new Comment or + // Label. + record.addLabelValue(label, sb.toString()); + sb.setLength(0); + label = null; + inValue = false; + } + } else { + // We're whitespace between label and value or whitespace + // before we've figured whether label or comment. + } + // Don't let the '\r' or CRLF through. + continue; + } + + if (inComment) { + continue; + } else if (inLabel) { + if (c == Label.COLON) { + label = sb.toString(); + sb.setLength(0); + inLabel = false; + continue; + } + } else { + if (!inLabel && !inValue && !inComment) { + // We have no state. Figure one. + if (Character.isWhitespace(c)) { + // If no state, and whitespace, skip. Don't record. + continue; + } else if (label == null && c == '#') { + inComment = true; + // Don't record comments. + continue; + } else if (label == null) { + inLabel = true; + } else { + inValue = true; + } + } + } + sb.append(c); + } + return record; + } + + /** + * @return Count of ANVLRecord bytes. Be careful, an empty ANVLRecord is + * CRLFCRLF so is of size 4. Also, expensive, since it makes String of + * the record so it can count bytes. + */ + public synchronized int getLength() { + int length = -1; + try { + length = getUTF8Bytes().length; + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + return length; + } + + public static boolean isCROrLF(final char c) { + return isCR(c) || isLF(c); + } + + public static boolean isCR(final char c) { + return c == ANVLRecord.CRLF.charAt(0); + } + + public static boolean isLF(final char c) { + return c == ANVLRecord.CRLF.charAt(1); + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/util/anvl/ANVLRecords.java b/commons/src/main/java/org/archive/util/anvl/ANVLRecords.java new file mode 100644 index 00000000..8ed5f43c --- /dev/null +++ b/commons/src/main/java/org/archive/util/anvl/ANVLRecords.java @@ -0,0 +1,61 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.util.anvl; + +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; + +import org.archive.io.UTF8Bytes; + +/** + * List of {@link ANVLRecord}s. + * @author stack + * @version $Date$ $Version$ + */ +public class ANVLRecords extends ArrayList implements UTF8Bytes { + private static final long serialVersionUID = 5361551920550106113L; + + public ANVLRecords() { + super(); + } + + public ANVLRecords(int initialCapacity) { + super(initialCapacity); + } + + public ANVLRecords(Collection c) { + super(c); + } + + public byte[] getUTF8Bytes() throws UnsupportedEncodingException { + return toString().getBytes(UTF8); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for (final Iterator i = iterator(); i.hasNext();) { + sb.append(i.next().toString()); + } + return super.toString(); + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/util/anvl/Element.java b/commons/src/main/java/org/archive/util/anvl/Element.java new file mode 100644 index 00000000..525c1fe8 --- /dev/null +++ b/commons/src/main/java/org/archive/util/anvl/Element.java @@ -0,0 +1,79 @@ +/* Element +* +* $Id$ +* +* Created on July 26, 2006. +* +* Copyright (C) 2006 Internet Archive.e. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util.anvl; + + +/** + * ANVL 'data element'. + * Made of a lone {@link Label}, or a {@link Label} plus {@link Value}. + * + * @author stack + * @see A Name-Value + * Language (ANVL) + */ +public class Element { + private final SubElement [] subElements; + + public Element(final Label l) { + this.subElements = new SubElement [] {l}; + } + + public Element(final Label l, final Value v) { + this.subElements = new SubElement [] {l, v}; + } + + public boolean isValue() { + return this.subElements.length > 1; + } + + public Label getLabel() { + return (Label)this.subElements[0]; + } + + public Value getValue() { + if (!isValue()) { + return null; + } + return (Value)this.subElements[1]; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < subElements.length; i++) { + sb.append(subElements[i].toString()); + if (i == 0) { + // Add colon after Label. + sb.append(':'); + if (isValue()) { + // Add space to intro the value. + sb.append(' '); + } + } + } + return sb.toString(); + } +} diff --git a/commons/src/main/java/org/archive/util/anvl/Label.java b/commons/src/main/java/org/archive/util/anvl/Label.java new file mode 100644 index 00000000..fdadb735 --- /dev/null +++ b/commons/src/main/java/org/archive/util/anvl/Label.java @@ -0,0 +1,41 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.util.anvl; + +class Label extends SubElement { + public static final char COLON = ':'; + + @SuppressWarnings("unused") + private Label() { + this(null); + } + + public Label(final String s) { + super(s); + } + + @Override + protected void checkCharacter(char c, String srcStr, int index) { + super.checkCharacter(c, srcStr, index); + if (c == COLON) { + throw new IllegalArgumentException("Label cannot contain " + COLON); + } + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/util/anvl/SubElement.java b/commons/src/main/java/org/archive/util/anvl/SubElement.java new file mode 100644 index 00000000..6e9e7c86 --- /dev/null +++ b/commons/src/main/java/org/archive/util/anvl/SubElement.java @@ -0,0 +1,84 @@ +/* SubElement +* +* $Id$ +* +* Created on July 26, 2006. +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util.anvl; + +/** + * Abstract ANVL 'data element' sub-part. + * Subclass to make a Comment, a Label, or a Value. + * @author stack + */ +abstract class SubElement { + private final String e; + + protected SubElement() { + this(null); + } + + public SubElement(final String s) { + this.e = baseCheck(s); + } + + protected String baseCheck(final String s) { + // Check for null. + if (s == null || s.length() <= 0) { + throw new IllegalArgumentException("Can't be null or empty"); + } + // Check for CRLF. + for (int i = 0; i < s.length(); i++) { + checkCharacter(s.charAt(i), s, i); + } + return s; + } + + protected void checkCharacter(final char c, final String srcStr, + final int index) { + checkControlCharacter(c, srcStr, index); + checkCRLF(c, srcStr, index); + } + + protected void checkControlCharacter(final char c, final String srcStr, + final int index) { + if (Character.isISOControl(c) && !Character.isWhitespace(c) || + !Character.isValidCodePoint(c)) { + throw new IllegalArgumentException(srcStr + + " contains a control character(s) or invalid code point: 0x" + + Integer.toHexString(c)); + } + } + + protected void checkCRLF(final char c, final String srcStr, + final int index) { + if (ANVLRecord.isCROrLF(c)) { + throw new IllegalArgumentException(srcStr + + " contains disallowed CRLF control character(s): 0x" + + Integer.toHexString(c)); + } + } + + @Override + public String toString() { + return e; + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/util/anvl/Value.java b/commons/src/main/java/org/archive/util/anvl/Value.java new file mode 100644 index 00000000..2a650ba2 --- /dev/null +++ b/commons/src/main/java/org/archive/util/anvl/Value.java @@ -0,0 +1,71 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.util.anvl; + +/** + * TODO: Now values 'fold' but should but perhaps they shouldn't be stored + * folded. Only when we serialize should we fold (But how to know where + * to fold?). + * @author stack + * @version $Date$ $Version$ + */ +class Value extends SubElement { + + private StringBuilder sb; + private boolean folding = false; + + @SuppressWarnings("unused") + private Value() { + this(null); + } + + public Value(final String s) { + super(s); + } + + protected String baseCheck(String s) { + this.sb = new StringBuilder(s.length() * 2); + super.baseCheck(s); + return sb.toString(); + } + + @Override + protected void checkCharacter(char c, String srcStr, int index) { + checkControlCharacter(c, srcStr, index); + // Now, rewrite the value String with folding (If CR or LF or CRLF + // present. + if (ANVLRecord.isCR(c)) { + this.folding = true; + this.sb.append(ANVLRecord.FOLD_PREFIX); + } else if (ANVLRecord.isLF(c)) { + if (!this.folding) { + this.folding = true; + this.sb.append(ANVLRecord.FOLD_PREFIX); + } else { + // Previous character was a CR. Fold prefix has been added. + } + } else if (this.folding && Character.isWhitespace(c)) { + // Only write out one whitespace character. Skip. + } else { + this.folding = false; + this.sb.append(c); + } + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/util/anvl/package.html b/commons/src/main/java/org/archive/util/anvl/package.html new file mode 100644 index 00000000..4a2a8963 --- /dev/null +++ b/commons/src/main/java/org/archive/util/anvl/package.html @@ -0,0 +1,42 @@ + + + +org.archive.util.anvl package + + +Parsers and Writers for the (expired) Internet-Draft A Name-Value +Language (ANVL). Use {@link org.archive.util.anvl.ANVLRecord} +to create new instances of ANVL Records and for parsing. + +

Implementation Details

+

The ANVL Internet-Draft of 14 February, 2005 is inspecific as to the +definition of 'blank line' and 'newline'. This parser implementation +assumes CRNL. +

+

Says "An element consists of a label, a colon, and an optional value". +Should that be: "An element consists of a label and an optional value, or a +comment."

+ +

Specification is unclear regards CR or NL in label or +comment (This implementation disallows CR or NL in labels but lets +them pass in comments).

+ +

A grammar would help. Here is RFC822: +

+     field       =  field-name ":" [ field-body ] CRLF
+     
+     field-name  =  1*<any CHAR, excluding CTLs, SPACE, and ":">
+     
+     field-body  =  field-body-contents
+                    [CRLF LWSP-char field-body]
+     
+     field-body-contents =
+                   <the ASCII characters making up the field-body, as
+                    defined in the following sections, and consisting
+                    of combinations of atom, quoted-string, and
+                    specials tokens, or else consisting of texts>
+
+

+ + diff --git a/commons/src/main/java/org/archive/util/bdbje/EnhancedEnvironment.java b/commons/src/main/java/org/archive/util/bdbje/EnhancedEnvironment.java new file mode 100644 index 00000000..821e833a --- /dev/null +++ b/commons/src/main/java/org/archive/util/bdbje/EnhancedEnvironment.java @@ -0,0 +1,87 @@ +/* EnhancedEnvironment.java + * + * Created on February 18. 2007 + * + * Copyright (C) 2007 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.util.bdbje; + +import java.io.File; + +import com.sleepycat.bind.serial.StoredClassCatalog; +import com.sleepycat.je.Database; +import com.sleepycat.je.DatabaseConfig; +import com.sleepycat.je.DatabaseException; +import com.sleepycat.je.Environment; +import com.sleepycat.je.EnvironmentConfig; + +/** + * Version of BDB_JE Environment with additional convenience features, such as + * a shared, cached StoredClassCatalog. (Additional convenience caching of + * Databases and StoredCollections may be added later.) + * + * @author gojomo + */ +public class EnhancedEnvironment extends Environment { + StoredClassCatalog classCatalog; + Database classCatalogDB; + + /** + * Constructor + * + * @param envHome directory in which to open environment + * @param envConfig config options + * @throws DatabaseException + */ + public EnhancedEnvironment(File envHome, EnvironmentConfig envConfig) throws DatabaseException { + super(envHome, envConfig); + } + + /** + * Return a StoredClassCatalog backed by a Database in this environment, + * either pre-existing or created (and cached) if necessary. + * + * @return the cached class catalog + */ + public StoredClassCatalog getClassCatalog() { + if(classCatalog == null) { + DatabaseConfig dbConfig = new DatabaseConfig(); + dbConfig.setAllowCreate(true); + try { + classCatalogDB = openDatabase(null, "classCatalog", dbConfig); + classCatalog = new StoredClassCatalog(classCatalogDB); + } catch (DatabaseException e) { + // TODO Auto-generated catch block + throw new RuntimeException(e); + } + } + return classCatalog; + } + + @Override + public synchronized void close() throws DatabaseException { + if(classCatalogDB!=null) { + classCatalogDB.close(); + } + super.close(); + } + + + +} diff --git a/commons/src/main/java/org/archive/util/fingerprint/ArrayLongFPCache.java b/commons/src/main/java/org/archive/util/fingerprint/ArrayLongFPCache.java new file mode 100644 index 00000000..b14e22aa --- /dev/null +++ b/commons/src/main/java/org/archive/util/fingerprint/ArrayLongFPCache.java @@ -0,0 +1,109 @@ +/* ArrayLongFPCache +* +* $Id$ +* +* Created on Oct 5, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util.fingerprint; + +/** + * Simple long fingerprint cache using a backing array; any long maps to + * one of 'smear' slots. Longs inserted should be randomly distributed, + * + * @author gojomo + */ +public class ArrayLongFPCache implements LongFPSet { + public static final int DEFAULT_CAPACITY = 1 << 20; // 1 million, 8MB + public static final int DEFAULT_SMEAR = 5; + + long cache[] = new long[DEFAULT_CAPACITY]; + int smear = DEFAULT_SMEAR; + int count = 0; + + public void setCapacity(int newCapacity) { + long[] oldCache = cache; + cache = new long[newCapacity]; + for(int i=0;itrue if set has changed with this addition + */ + boolean add(long l); + + /** + * Does this set contain a given fingerprint. + * @param l the fingerprint to check for + * @return true if the fingerprint is in the set + */ + boolean contains(long l); + + /** + * Remove a fingerprint from the set, if it is there + * @param l the fingerprint to remove + * @return true if we removed the fingerprint + */ + boolean remove(long l); + + /** get the number of elements in the Set + * @return the number of elements in the Set + */ + long count(); + + /** + * Do a contains() check that doesn't require laggy + * activity (eg disk IO). If this returns true, + * fp is definitely contained; if this returns + * false, fp *MAY* still be contained -- must use + * full-cost contains() to be sure. + * + * @param fp the fingerprint to check for + * @return true if contains the fingerprint + */ + boolean quickContains(long fp); +} diff --git a/commons/src/main/java/org/archive/util/fingerprint/LongFPSetCache.java b/commons/src/main/java/org/archive/util/fingerprint/LongFPSetCache.java new file mode 100644 index 00000000..68664cb7 --- /dev/null +++ b/commons/src/main/java/org/archive/util/fingerprint/LongFPSetCache.java @@ -0,0 +1,77 @@ +/* Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * LongFPSetCache.java + * Created on Oct 21, 2003 + * + * $Header$ + */ +package org.archive.util.fingerprint; + + + +/** + * Like a MemLongFPSet, but with fixed capacity and maximum size. + * When an add would expand past the maximum size, an old entry + * is deleted via a clock/counter algorithm. + * + * @author gojomo + * + */ +public class LongFPSetCache extends MemLongFPSet { + + private static final long serialVersionUID = -5307436423975825566L; + + long sweepHand = 0; + + public LongFPSetCache() { + super(); + } + + public LongFPSetCache(int capacityPowerOfTwo, float loadFactor) { + super(capacityPowerOfTwo, loadFactor); + } + + protected void noteAccess(long index) { + if(slots[(int)index]0) { + if(slots[(int)sweepHand]==0) { + removeAt(sweepHand); + toDiscard--; + } else { + if (slots[(int)sweepHand]>0) { + slots[(int)sweepHand]--; + } + } + sweepHand++; + if (sweepHand==slots.length) { + sweepHand = 0; + } + } + } +} diff --git a/commons/src/main/java/org/archive/util/fingerprint/LongFPSetTestCase.java b/commons/src/main/java/org/archive/util/fingerprint/LongFPSetTestCase.java new file mode 100644 index 00000000..6d12fbb9 --- /dev/null +++ b/commons/src/main/java/org/archive/util/fingerprint/LongFPSetTestCase.java @@ -0,0 +1,134 @@ +/* LongFPSetTestCase + * + * $Id$ + * + * Created Wed Jan 21 09:00:29 CET 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +package org.archive.util.fingerprint; + +import junit.framework.TestCase; + +/** + * JUnit test suite for LongFPSet. This is an abstract class which defines + * the generic tests that test the {@link LongFPSet} interface. Subclasses + * may test specifics of {@link LongFPSet} subclass implementations + * + * @author James Casey + * @version $ Id:$ + */ +abstract public class LongFPSetTestCase extends TestCase { + + /** the unerlying FPSet we wish to test */ + private LongFPSet fpSet; + + /** + * Create a new LongFPSetTest object + * + * @param testName the name of the test + */ + public LongFPSetTestCase(final String testName) { + super(testName); + } + + public void setUp() { + fpSet = makeLongFPSet(); + } + + abstract LongFPSet makeLongFPSet(); + + /** check that we can add fingerprints */ + public void testAdd() { + long l1 = (long)1234; + long l2 = (long)2345; + + assertEquals("empty set to start", 0, fpSet.count()); + assertTrue("set changed on addition of l1", fpSet.add(l1)); + assertTrue("set changed on addition of l2", fpSet.add(l2)); + assertFalse("set didn't change on re-addition of l1", fpSet.add(l1)); + } + + /** check we can call add/remove/contains() with 0 as a value */ + public void testWithZero() { + long zero = (long)0; + + assertEquals("empty set to start", 0, fpSet.count()); + assertFalse("zero is not there", fpSet.contains(zero)); + assertTrue("zero added", fpSet.add(zero)); + + // now one element + assertEquals("one fp in set", 1, fpSet.count()); + assertTrue("zero is the element", fpSet.contains(zero)); + + // and remove + assertTrue("zero removed", fpSet.remove(zero)); + assertEquals("empty set again", 0, fpSet.count()); + } + + /** check that contains() does what we expect */ + public void testContains() { + long l1 = (long) 1234; + long l2 = (long) 2345; + long l3 = (long) 1334; + + assertEquals("empty set to start", 0, fpSet.count()); + fpSet.add(l1); + fpSet.add(l2); + + assertTrue("contains l1", fpSet.contains(l1)); + assertTrue("contains l2", fpSet.contains(l2)); + assertFalse("does not contain l3", fpSet.contains(l3)); + } + + /** test remove() works as expected */ + public void testRemove() { + long l1 = (long) 1234; + + assertEquals("empty set to start", 0, fpSet.count()); + + // remove before it's there + assertFalse("fp not in set", fpSet.remove(l1)); + // now add + fpSet.add(l1); + // and remove again + assertTrue("fp was in set", fpSet.remove(l1)); + // check set is empty again + assertEquals("empty set again", 0, fpSet.count()); + } + + /** check count works ok */ + public void testCount() { + final int NUM = 1000; + assertEquals("empty set to start", 0, fpSet.count()); + + for(int i = 1; i < NUM; ++i) { + fpSet.add((long)i); + assertEquals("correct num", i, fpSet.count()); + } + for (int i = NUM - 1; i > 0; --i) { + fpSet.remove((long) i); + assertEquals("correct num", i -1, fpSet.count()); + } + assertEquals("empty set to start", 0, fpSet.count()); + + } +} + diff --git a/commons/src/main/java/org/archive/util/fingerprint/MemLongFPSet.java b/commons/src/main/java/org/archive/util/fingerprint/MemLongFPSet.java new file mode 100644 index 00000000..681dfc7c --- /dev/null +++ b/commons/src/main/java/org/archive/util/fingerprint/MemLongFPSet.java @@ -0,0 +1,122 @@ +/* MemLongSet + * + * $Id$ + * + * Created on Oct 19, 2003 + * + * Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.util.fingerprint; + +import java.io.Serializable; +import java.util.logging.Logger; + +import org.archive.util.AbstractLongFPSet; + +/** + * Open-addressing in-memory hash set for holding primitive long fingerprints. + * + * @author Gordon Mohr + */ +public class MemLongFPSet extends AbstractLongFPSet +implements LongFPSet, Serializable { + + + private static final long serialVersionUID = -4301879539092625698L; + + + private static Logger logger = + Logger.getLogger(MemLongFPSet.class.getName()); + private static final int DEFAULT_CAPACITY_POWER_OF_TWO = 10; + private static final float DEFAULT_LOAD_FACTOR = 0.75f; + protected byte[] slots; + protected long[] values; + + public MemLongFPSet() { + this(DEFAULT_CAPACITY_POWER_OF_TWO, DEFAULT_LOAD_FACTOR); + } + + /** + * @param capacityPowerOfTwo The capacity as the exponent of a power of 2. + * e.g if the capacity is 4 this means 2^^4 + * entries. + * @param loadFactor The load factor as a fraction. This gives the amount + * of free space to keep in the Set. + */ + public MemLongFPSet(int capacityPowerOfTwo, float loadFactor) { + super(capacityPowerOfTwo, loadFactor); + slots = new byte[1 << capacityPowerOfTwo]; + for(int i = 0; i < (1 << capacityPowerOfTwo); i++) { + slots[i] = EMPTY; // flag value for unused + } + values = new long[1 << capacityPowerOfTwo]; + } + + protected void setAt(long i, long val) { + slots[(int)i] = 1; + values[(int)i] = val; + } + + protected long getAt(long i) { + return values[(int)i]; + } + + protected void makeSpace() { + grow(); + } + + private void grow() { + // Catastrophic event. Log its occurance. + logger.info("Doubling fingerprinting slots to " + + (1 << this.capacityPowerOfTwo)); + long[] oldValues = values; + byte[] oldSlots = slots; + capacityPowerOfTwo++; + values = new long[1 << capacityPowerOfTwo]; + slots = new byte[1 << capacityPowerOfTwo]; + for(int i = 0; i < (1 << capacityPowerOfTwo); i++) { + slots[i]=EMPTY; // flag value for unused + } + count=0; + for(int i = 0; i< oldValues.length; i++) { + if(oldSlots[i]>=0) { + add(oldValues[i]); + } + } + } + + protected void relocate(long val, long oldIndex, long newIndex) { + values[(int)newIndex] = values[(int)oldIndex]; + slots[(int)newIndex] = slots[(int)oldIndex]; + slots[(int)oldIndex] = EMPTY; + } + + protected int getSlotState(long i) { + return slots[(int)i]; + } + + protected void clearAt(long index) { + slots[(int)index]=EMPTY; + values[(int)index]=0; + } + + public boolean quickContains(long fp) { + return contains(fp); + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/util/iterator/CompositeIterator.java b/commons/src/main/java/org/archive/util/iterator/CompositeIterator.java new file mode 100644 index 00000000..2d6fee29 --- /dev/null +++ b/commons/src/main/java/org/archive/util/iterator/CompositeIterator.java @@ -0,0 +1,111 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.util.iterator; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * An iterator that's built up out of any number of other iterators. + * @author gojomo + */ +public class CompositeIterator implements Iterator { + ArrayList> iterators = new ArrayList>(); + Iterator currentIterator; + int indexOfCurrentIterator = -1; + + /** + * Moves to the next (non empty) iterator. Returns false if there are no + * more (non empty) iterators, true otherwise. + * @return false if there are no more (non empty) iterators, true otherwise. + */ + private boolean nextIterator() { + if (++indexOfCurrentIterator < iterators.size()) { + currentIterator = iterators.get(indexOfCurrentIterator); + // If the new iterator was empty this will move us to the next one. + return hasNext(); + } else { + currentIterator = null; + return false; + } + } + + /* (non-Javadoc) + * @see java.util.Iterator#hasNext() + */ + public boolean hasNext() { + if(currentIterator!=null && currentIterator.hasNext()) { + // Got more + return true; + } else { + // Have got more if we can queue up a new iterator. + return nextIterator(); + } + } + + /* (non-Javadoc) + * @see java.util.Iterator#next() + */ + public E next() { + if(hasNext()) { + return currentIterator.next(); + } else { + throw new NoSuchElementException(); + } + } + + /* (non-Javadoc) + * @see java.util.Iterator#remove() + */ + public void remove() { + throw new UnsupportedOperationException(); + } + + /** + * Create an empty CompositeIterator. Internal + * iterators may be added later. + */ + public CompositeIterator() { + super(); + } + + /** + * Convenience method for concatenating together + * two iterators. + * @param i1 + * @param i2 + */ + public CompositeIterator(Iterator i1, Iterator i2) { + this(); + add(i1); + add(i2); + } + + /** + * Add an iterator to the internal chain. + * + * @param i an iterator to add. + */ + public void add(Iterator i) { + iterators.add(i); + } + +} diff --git a/commons/src/main/java/org/archive/util/iterator/LineReadingIterator.java b/commons/src/main/java/org/archive/util/iterator/LineReadingIterator.java new file mode 100644 index 00000000..ff1fcc46 --- /dev/null +++ b/commons/src/main/java/org/archive/util/iterator/LineReadingIterator.java @@ -0,0 +1,65 @@ +/* LineReadingIterator +* +* $Id$ +* +* Created on Jul 27, 2004 +* +* Copyright (C) 2004 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util.iterator; + +import java.io.BufferedReader; +import java.io.IOException; +import java.util.logging.Logger; + +/** + * Utility class providing an Iterator interface over line-oriented + * text input, as a thin wrapper over a BufferedReader. + * + * @author gojomo + */ +public class LineReadingIterator extends LookaheadIterator { + private static final Logger logger = + Logger.getLogger(LineReadingIterator.class.getName()); + + protected BufferedReader reader = null; + + public LineReadingIterator(BufferedReader r) { + reader = r; + } + + /** + * Loads next line into lookahead spot + * + * @return whether any item was loaded into next field + */ + protected boolean lookahead() { + try { + next = this.reader.readLine(); + if(next == null) { + // TODO: make this close-on-exhaust optional? + reader.close(); + } + return (next!=null); + } catch (IOException e) { + logger.warning(e.toString()); + return false; + } + } +} diff --git a/commons/src/main/java/org/archive/util/iterator/LookaheadIterator.java b/commons/src/main/java/org/archive/util/iterator/LookaheadIterator.java new file mode 100644 index 00000000..60b79f13 --- /dev/null +++ b/commons/src/main/java/org/archive/util/iterator/LookaheadIterator.java @@ -0,0 +1,79 @@ +/* TransformingIteratorWrapper +* +* $Id$ +* +* Created on Mar 25, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util.iterator; + +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * Superclass for Iterators which must probe ahead to know if + * a 'next' exists, and thus have a cached next between a call + * to hasNext() and next(). + * + * @author gojomo + * + */ +public abstract class LookaheadIterator implements Iterator { + protected T next; + + /** + * Test whether any items remain; loads next item into + * holding 'next' field. + * + * @see java.util.Iterator#hasNext() + */ + public boolean hasNext() { + return (this.next != null)? true: lookahead(); + } + + /** + * Caches the next item if available. + * + * @return true if there was a next item to cache, false otherwise + */ + protected abstract boolean lookahead(); + + /** + * Return the next item. + * + * @see java.util.Iterator#next() + */ + public T next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + // 'next' is guaranteed non-null by a hasNext() which returned true + T returnObj = this.next; + this.next = null; + return returnObj; + } + + /* (non-Javadoc) + * @see java.util.Iterator#remove() + */ + public void remove() { + throw new UnsupportedOperationException(); + } +} diff --git a/commons/src/main/java/org/archive/util/iterator/RegexpLineIterator.java b/commons/src/main/java/org/archive/util/iterator/RegexpLineIterator.java new file mode 100644 index 00000000..c836d470 --- /dev/null +++ b/commons/src/main/java/org/archive/util/iterator/RegexpLineIterator.java @@ -0,0 +1,96 @@ +/* LineReadingIterator +* +* $Id$ +* +* Created on Jul 27, 2004 +* +* Copyright (C) 2004 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util.iterator; + +import java.util.Iterator; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Utility class providing an Iterator interface over line-oriented + * text input. By providing regexps indicating lines to ignore + * (such as pure whitespace or comments), lines to consider input, and + * what to return from the input lines (such as a whitespace-trimmed + * non-whitespace token with optional trailing comment), this can + * be configured to handle a number of formats. + * + * The public static members provide pattern configurations that will + * be helpful in a wide variety of contexts. + * + * @author gojomo + */ +public class RegexpLineIterator +extends TransformingIteratorWrapper { + private static final Logger logger = + Logger.getLogger(RegexpLineIterator.class.getName()); + + public static final String COMMENT_LINE = "\\s*(#.*)?"; + public static final String NONWHITESPACE_ENTRY_TRAILING_COMMENT = + "^\\s*(\\S+)\\s*(#.*)?$"; + public static final String TRIMMED_ENTRY_TRAILING_COMMENT = + "^\\s*([^#]+?)\\s*(#.*)?$"; + + public static final String ENTRY = "$1"; + + protected Matcher ignoreLine = null; + protected Matcher extractLine = null; + protected String outputTemplate = null; + + + public RegexpLineIterator(Iterator inner, String ignore, + String extract, String replace) { + this.inner = inner; + ignoreLine = Pattern.compile(ignore).matcher(""); + extractLine = Pattern.compile(extract).matcher(""); + outputTemplate = replace; + } + + /** + * Loads next item into lookahead spot, if available. Skips + * lines matching ignoreLine; extracts desired portion of + * lines matching extractLine; informationally reports any + * lines matching neither. + * + * @return whether any item was loaded into next field + */ + protected String transform(String line) { + ignoreLine.reset(line); + if(ignoreLine.matches()) { + return null; + } + extractLine.reset(line); + if(extractLine.matches()) { + StringBuffer output = new StringBuffer(); + // TODO: consider if a loop that find()s all is more + // generally useful here + extractLine.appendReplacement(output,outputTemplate); + return output.toString(); + } + // no match; possibly error + logger.warning("line not extracted nor no-op: "+line); + return null; + } +} diff --git a/commons/src/main/java/org/archive/util/iterator/TransformingIteratorWrapper.java b/commons/src/main/java/org/archive/util/iterator/TransformingIteratorWrapper.java new file mode 100644 index 00000000..fdfef41c --- /dev/null +++ b/commons/src/main/java/org/archive/util/iterator/TransformingIteratorWrapper.java @@ -0,0 +1,71 @@ +/* TransformingIteratorWrapper +* +* $Id$ +* +* Created on Mar 25, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util.iterator; + +import java.util.Iterator; + +/** + * Superclass for Iterators which transform and/or filter results + * from a wrapped Iterator. Because transform() has the option of + * discarding an item from the inner Iterator (by returning null), + * this is a kind of LookaheadIterator. + * + * @author gojomo + */ +public abstract class TransformingIteratorWrapper +extends LookaheadIterator { + protected Iterator inner; + + + /* (non-Javadoc) + * @see org.archive.util.iterator.LookaheadIterator#lookahead() + */ + protected boolean lookahead() { + assert next == null : "looking ahead when next is already loaded"; + while(inner.hasNext()) { + next = transform(inner.next()); + if(next!=null) { + return true; + } + } + noteExhausted(); + return false; + } + + /** + * Any cleanup to occur when hasNext() is about to return false + */ + protected void noteExhausted() { + // by default, do nothing + + } + + /** + * @param object Object to transform. + * @return Transfomed object. + */ + protected abstract Transformed transform(Original object); + +} diff --git a/commons/src/main/java/org/archive/util/ms/BlockFileSystem.java b/commons/src/main/java/org/archive/util/ms/BlockFileSystem.java new file mode 100644 index 00000000..3aaec485 --- /dev/null +++ b/commons/src/main/java/org/archive/util/ms/BlockFileSystem.java @@ -0,0 +1,73 @@ +/* BlockFileSystem +* +* Created on September 12, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util.ms; + +import java.io.IOException; + +import org.archive.io.SeekInputStream; + + +/** + * Describes the internal file system contained in .doc files. + */ +public interface BlockFileSystem { + + + /** + * The size of a block in bytes. + */ + int BLOCK_SIZE = 512; + + + /** + * Returns the root entry of the file system. Subfiles and directories + * can be found by searching the returned entry. + * + * @return the root entry + * @throws IOException if an IO error occurs + */ + public abstract Entry getRoot() throws IOException; + + + /** + * Returns the number of the block that follows the given block. + * The internal block allocation tables are consulted to determine the + * next block. A return value that is less than zero indicates that + * there is no next block. + * + * @param block the number of block whose successor to return + * @return the successor of that block + * @throws IOException if an IO error occurs + */ + public abstract int getNextBlock(int block) throws IOException; + + + /** + * Returns the raw input stream for this file system. + * Typically this will be the random access file containing the .doc. + * + * @return the raw input stream for this file system + */ + public abstract SeekInputStream getRawInput(); + +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/util/ms/BlockInputStream.java b/commons/src/main/java/org/archive/util/ms/BlockInputStream.java new file mode 100644 index 00000000..80a2ec04 --- /dev/null +++ b/commons/src/main/java/org/archive/util/ms/BlockInputStream.java @@ -0,0 +1,212 @@ +/* BlockInputStream +* +* Created on September 12, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util.ms; + +import java.io.IOException; + +import org.archive.io.SeekInputStream; +import static org.archive.util.ms.BlockFileSystem.BLOCK_SIZE; + + +/** + * InputStream for a file contained in a BlockFileSystem. + */ +public class BlockInputStream extends SeekInputStream { + + + /** + * The starting block number. + */ + private int start; + + + /** + * The current block. + */ + private int block; + + + /** + * The BlockFileSystem that produced this stream. + */ + private BlockFileSystem bfs; + + + /** + * The raw input stream of the BlockFileSystem. + */ + private SeekInputStream raw; + + + /** + * The current logical position of this stream. + */ + private long position; + + + /** + * The current file pointer position of the raw input stream. + */ + private long expectedRawPosition; + + + /** + * The number of bytes read in the current block. + */ + private int blockBytesRead; + + + /** + * Constructor. + * + * @param bfs The block file system that owns this stream + * @param block The starting block number. + */ + public BlockInputStream(BlockFileSystem bfs, int block) throws IOException { + this.raw = bfs.getRawInput(); + this.bfs = bfs; + this.start = block; + this.block = block; + this.position = 0; + seek(block, 0); + } + + + + private void seek(long block, long rem) throws IOException { + assert rem < BLOCK_SIZE; + long pos = (block + 1) * BLOCK_SIZE + rem; + blockBytesRead = (int)rem; + expectedRawPosition = pos; + raw.position(pos); + } + + + private void ensureRawPosition() throws IOException { + if (raw.position() != expectedRawPosition) { + raw.position(expectedRawPosition); + } + } + + private boolean ensureBuffer() throws IOException { + if (block < 0) { + return false; + } + ensureRawPosition(); + if (blockBytesRead < BLOCK_SIZE) { + return true; + } + block = bfs.getNextBlock(block); + if (block < 0) { + return false; + } + seek(block, 0); + return true; + } + + + public long skip(long v) throws IOException { + // FIXME + int r = read(); + return (r < 0) ? 0 : 1; + } + + public int read() throws IOException { + if (!ensureBuffer()) { + return -1; + } + int r = raw.read(); + position++; + expectedRawPosition++; + blockBytesRead++; + return r; + } + + + public int read(byte[] b, int ofs, int len) throws IOException { + if (!ensureBuffer()) { + return 0; + } + int rem = BLOCK_SIZE - (int)(position % BLOCK_SIZE); + len = Math.min(len, rem); + int c = raw.read(b, ofs, len); + position += c; + expectedRawPosition += c; + blockBytesRead++; + return len; + } + + + public int read(byte[] b) throws IOException { + return read(b, 0, b.length); + } + + + public long position() { + return position; + } + + + public void position(long v) throws IOException { + ensureRawPosition(); + if (v == position) { + return; + } + + // If new position is in same block, just seek. + if (v / BLOCK_SIZE == position / BLOCK_SIZE) { + long rem = v % BLOCK_SIZE; + seek(block, rem); + position = v; + return; + } + + if (v > position) { + seekAfter(v); + } else { + seekBefore(v); + } + } + + + private void seekAfter(long v) throws IOException { + long currentBlock = position / BLOCK_SIZE; + long destBlock = v / BLOCK_SIZE; + long blockAdvance = destBlock - currentBlock; + for (int i = 0; i < blockAdvance; i++) { + block = bfs.getNextBlock(block); + } + seek(block, v % BLOCK_SIZE); + position = v; + } + + + private void seekBefore(long v) throws IOException { + long blockAdvance = (v - 1) / BLOCK_SIZE; + block = start; + for (int i = 0; i < blockAdvance; i++) { + block = bfs.getNextBlock(block); + } + seek(block, v % BLOCK_SIZE); + } +} diff --git a/commons/src/main/java/org/archive/util/ms/Cp1252.java b/commons/src/main/java/org/archive/util/ms/Cp1252.java new file mode 100644 index 00000000..3702b4b4 --- /dev/null +++ b/commons/src/main/java/org/archive/util/ms/Cp1252.java @@ -0,0 +1,87 @@ +/* Cp1252 +* +* Created on September 12, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util.ms; + + +import java.io.UnsupportedEncodingException; + + +/** + * A fast implementation of code page 1252. This is used to convert bytes + * to characters in .doc files that don't use unicode. + * + *

The Java Charset APIs seemed like overkill for these translations, + * since 1 byte always translates into 1 character. + * + * @author pjack + */ +public class Cp1252 { + + + /** + * The translation table. If x is an unsigned byte from a .doc + * text stream, then XLAT[x] is the Unicode character that byte + * represents. + */ + final private static char[] XLAT = createTable(); + + + /** + * Static utility library, do not instantiate. + */ + private Cp1252() { + } + + + /** + * Generates the translation table. The Java String API is used for each + * possible byte to determine the corresponding Unicode character. + * + * @return the Cp1252 translation table + */ + private static char[] createTable() { + char[] result = new char[256]; + byte[] b = new byte[1]; + for (int i = 0; i < 256; i++) try { + b[0] = (byte)i; + String s = new String(b, "Cp1252"); + result[i] = s.charAt(0); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + return result; + } + + + /** + * Returns the Unicode character for the given Cp1252 byte. + * + * @param b an unsigned byte from 0 to 255 + * @return the Unicode character corresponding to that byte + */ + public static char decode(int b) { + return XLAT[b]; + } + + +} diff --git a/commons/src/main/java/org/archive/util/ms/DefaultBlockFileSystem.java b/commons/src/main/java/org/archive/util/ms/DefaultBlockFileSystem.java new file mode 100644 index 00000000..69dc1136 --- /dev/null +++ b/commons/src/main/java/org/archive/util/ms/DefaultBlockFileSystem.java @@ -0,0 +1,287 @@ +/* DefaultBlockFileSystem +* +* Created on September 12, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util.ms; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Map; + +import org.archive.io.SeekInputStream; +import org.archive.util.IoUtils; +import org.archive.util.LRU; + + +/** + * Default implementation of the Block File System. + * + *

The overall structure of a BlockFileSystem file (such as a .doc file) is + * as follows. The file is divided into blocks, which are of uniform length + * (512 bytes). The first block (at file pointer 0) is called the header + * block. It's used to look up other blocks in the file. + * + *

Subfiles contained within the .doc file are organized using a Block + * Allocation Table, or BAT. The BAT is basically a linked list; given a + * block number, the BAT will tell you the next block number. Note that + * the header block has no number; block #0 is the first block after the + * header. Thus, to convert a block number to a file pointer: + * int filePointer = (blockNumber + 1) * BLOCK_SIZE. + * + *

The BAT itself is discontinuous, however. To find the blocks that + * comprise the BAT, you have to look in the header block. The header block + * contains an array of 109 pointers to the blocks that comprise the BAT. + * If more than 109 BAT blocks are required (in other words, if the .doc + * file is larger than ~6 megabytes), then something called the + * XBAT comes into play. + * + *

XBAT blocks contain pointers to the 110th BAT block and beyond. + * The first XBAT block is stored at a file pointer listed in the header. + * The other XBAT blocks are always stored in order after the first; the + * XBAT table is continuous. One is inclined to wonder why the BAT itself + * is not so stored, but oh well. + * + *

The BAT only tells you the next block for a given block. To find the + * first block for a subfile, you have to look up that subfile's directory + * entry. Each directory entry is a 128 byte structure in the file, so four + * of them fit in a block. The number of the first block of the entry list + * is stored in the header. To find subsequent entry blocks, the BAT must + * be used. + * + *

I'm telling you all this so that you understand the caching that this + * class provides. + * + *

First, directory entries are not cached. It's assumed that they will + * be looked up at the beginning of a lengthy operation, and then forgotten + * about. This is certainly the case for {@link Doc#getText(BlockFileSystem)}. + * If you need to remember directory entries, you can manually store the Entry + * objects in a map or something, as they don't grow stale. + * + *

This class keeps all 512 bytes of the header block in memory at all + * times. This prevents a potentially expensive file pointer repositioning + * every time you're trying to figure out what comes next. + * + *

BAT and XBAT blocks are stored in a least-recently used cache. The + * n most recent BAT and XBAT blocks are remembered, where n + * is set at construction time. The minimum value of n is 1. For small + * files, this can prevent file pointer repositioning for BAT look ups. + * + *

The BAT/XBAT cache only takes up memory as needed. If the specified + * cache size is 100 blocks, but the file only has 4 BAT blocks, then only + * 2048 bytes will be used by the cache. + * + *

Note this class only caches BAT and XBAT blocks. It does not cache the + * blocks that actually make up a subfile's contents. It is assumed that those + * blocks will only be accessed once per operation (again, this is what + * {Doc.getText(BlockFileSystem)} typically requires.) + * + * @author pjack + * @see http://jakarta.apache.org/poi/poifs/fileformat.html + */ +public class DefaultBlockFileSystem implements BlockFileSystem { + + + /** + * Pointers per BAT block. + */ + final private static int POINTERS_PER_BAT = 128; + + + /** + * Size of a BAT pointer in bytes. (In other words, 4). + */ + final private static int BAT_POINTER_SIZE = BLOCK_SIZE / POINTERS_PER_BAT; + + + /** + * The number of BAT pointers in the header block. After this many + * BAT blocks, the XBAT blocks must be consulted. + */ + final private static int HEADER_BAT_LIMIT = 109; + + + /** + * The size of an entry record in bytes. + */ + final private static int ENTRY_SIZE = 128; + + + /** + * The number of entries that can fit in a block. + */ + final private static int ENTRIES_PER_BLOCK = BLOCK_SIZE / ENTRY_SIZE; + + + /** + * The .doc file as a stream. + */ + private SeekInputStream input; + + + /** + * The header block. + */ + private HeaderBlock header; + + + /** + * Cache of BAT and XBAT blocks. + */ + private Map cache; + + + /** + * Constructor. + * + * @param input the file to read from + * @param batCacheSize number of BAT and XBAT blocks to cache + * @throws IOException if an IO error occurs + */ + public DefaultBlockFileSystem(SeekInputStream input, int batCacheSize) + throws IOException { + this.input = input; + byte[] temp = new byte[BLOCK_SIZE]; + IoUtils.readFully(input, temp); + this.header = new HeaderBlock(ByteBuffer.wrap(temp)); + this.cache = new LRU(batCacheSize); + } + + + public Entry getRoot() throws IOException { + // Position to the first block of the entry list. + int block = header.getEntriesStart(); + input.position((block + 1) * BLOCK_SIZE); + + // The root entry is always entry #0. + return new DefaultEntry(this, input, 0); + } + + + /** + * Returns the entry with the given number. + * + * @param entryNumber the number of the entry to return + * @return that entry, or null if no such entry exists + * @throws IOException if an IO error occurs + */ + Entry getEntry(int entryNumber) throws IOException { + // Entry numbers < 0 typically indicate an end-of-stream. + if (entryNumber < 0) { + return null; + } + + // It's impossible to check against the upper bound, because the + // upper bound is not recorded anywhere. + + // Advance to the block containing the desired entry. + int blockCount = entryNumber / ENTRIES_PER_BLOCK; + int remainder = entryNumber % ENTRIES_PER_BLOCK; + int block = header.getEntriesStart(); + for (int i = 0; i < blockCount; i++) { + block = getNextBlock(block); + } + + if (block < 0) { + // Given entry number exceeded the number of available entries. + return null; + } + + int filePos = (block + 1) * BLOCK_SIZE + remainder * ENTRY_SIZE; + input.position(filePos); + + return new DefaultEntry(this, input, entryNumber); + } + + + public int getNextBlock(int block) throws IOException { + if (block < 0) { + return block; + } + + // Index into the header array of BAT blocks. + int headerIndex = block / POINTERS_PER_BAT; + + // Index within that BAT block of the block we're interested in. + int batBlockIndex = block % POINTERS_PER_BAT; + + int batBlockNumber = batLookup(headerIndex); + ByteBuffer batBlock = getBATBlock(batBlockNumber); + return batBlock.getInt(batBlockIndex * BAT_POINTER_SIZE); + } + + + /** + * Looks up the block number of a BAT block. + * + * @param headerIndex + * @return + * @throws IOException + */ + private int batLookup(int headerIndex) throws IOException { + if (headerIndex < HEADER_BAT_LIMIT + 1) { + return header.getBATBlockNumber(headerIndex); + } + + // Find the XBAT block of interest + headerIndex -= HEADER_BAT_LIMIT; + int xbatBlockNumber = headerIndex / POINTERS_PER_BAT; + xbatBlockNumber += header.getExtendedBATStart(); + ByteBuffer xbat = getBATBlock(xbatBlockNumber); + + // Find the bat Block number inside the XBAT block + int xbatBlockIndex = headerIndex % POINTERS_PER_BAT; + return xbat.getInt(xbatBlockIndex * BAT_POINTER_SIZE); + } + + + /** + * Returns the BAT block with the given block number. + * If the BAT block were previously cached, then the cached version + * is returned. Otherwise, the file pointer is repoisitioned to + * the start of the given block, and the 512 bytes are read and + * stored in the cache. + * + * @param block the block number of the BAT block to return + * @return the BAT block + * @throws IOException + */ + private ByteBuffer getBATBlock(int block) throws IOException { + ByteBuffer r = cache.get(block); + if (r != null) { + return r; + } + + byte[] buf = new byte[BLOCK_SIZE]; + input.position((block + 1) * BLOCK_SIZE); + IoUtils.readFully(input, buf); + + r = ByteBuffer.wrap(buf); + r.order(ByteOrder.LITTLE_ENDIAN); + cache.put(block, r); + return r; + } + + + public SeekInputStream getRawInput() { + return input; + } +} diff --git a/commons/src/main/java/org/archive/util/ms/DefaultEntry.java b/commons/src/main/java/org/archive/util/ms/DefaultEntry.java new file mode 100644 index 00000000..a78ad328 --- /dev/null +++ b/commons/src/main/java/org/archive/util/ms/DefaultEntry.java @@ -0,0 +1,161 @@ +/* DefaultEntry +* +* Created on September 12, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util.ms; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.List; + +import org.archive.util.IoUtils; +import org.archive.io.SeekInputStream; + +class DefaultEntry implements Entry { + + + private DefaultBlockFileSystem origin; + private String name; + private EntryType type; + private int previous; + private int next; + private int child; + private int startBlock; + private int size; + private int index; + + + public DefaultEntry(DefaultBlockFileSystem origin, SeekInputStream input, int index) + throws IOException { + this.index = index; + // FIXME: Read directly from the stream + this.origin = origin; + byte[] temp = new byte[128]; + IoUtils.readFully(input, temp); + ByteBuffer buf = ByteBuffer.wrap(temp); + buf.order(ByteOrder.LITTLE_ENDIAN); + buf.position(0); + + StringBuilder nameBuf = new StringBuilder(); + + char ch = buf.getChar(); + while (ch != 0) { + nameBuf.append(ch); + ch = buf.getChar(); + } + this.name = nameBuf.toString(); + + byte typeFlag = buf.get(0x42); + switch (typeFlag) { + case 1: + this.type = EntryType.DIRECTORY; + break; + case 2: + this.type = EntryType.FILE; + break; + case 5: + this.type = EntryType.ROOT; + break; + default: + throw new IllegalStateException("Invalid type: " + typeFlag); + } + + this.previous = buf.getInt(0x44); + this.next = buf.getInt(0x48); + this.child = buf.getInt(0x4C); + this.startBlock = buf.getInt(0x74); + this.size = buf.getInt(0x78); + } + + + public String getName() { + return name; + } + + + public EntryType getType() { + return type; + } + + + public Entry getNext() throws IOException { + return origin.getEntry(next); + } + + + public Entry getPrevious() throws IOException { + return origin.getEntry(previous); + } + + + public Entry getChild() throws IOException { + return origin.getEntry(child); + } + + public SeekInputStream open() throws IOException { + return new BlockInputStream(origin, startBlock); + } + + + public List list() throws IOException { + if (child < 0) { + throw new IllegalStateException("Can't list non-directory."); + } + Entry child = getChild(); + ArrayList r = new ArrayList(); + list(r, child); + return r; + } + + + public static void list(List list, Entry e) throws IOException { + if (e == null) { + return; + } + list.add(e); + list(list, e.getPrevious()); + list(list, e.getNext()); + } + + + public int getIndex() { + return index; + } + + + public String toString() { + StringBuilder result = new StringBuilder("Entry{"); + result.append("name=").append(name); + result.append(" index=").append(index); + result.append(" type=").append(type); + result.append(" size=").append(size); + result.append(" prev=").append(previous); + result.append(" next=").append(next); + result.append(" child=").append(child); + result.append(" startBlock=").append(startBlock); + result.append("}"); + return result.toString(); + } + + +} diff --git a/commons/src/main/java/org/archive/util/ms/Doc.java b/commons/src/main/java/org/archive/util/ms/Doc.java new file mode 100644 index 00000000..203f1cd7 --- /dev/null +++ b/commons/src/main/java/org/archive/util/ms/Doc.java @@ -0,0 +1,161 @@ +/* Doc +* +* Created on September 12, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util.ms; + + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.archive.io.Endian; +import org.archive.io.RandomAccessInputStream; +import org.archive.io.SeekInputStream; +import org.archive.io.SeekReader; + + +/** + * Reads .doc files. + * + * @author pjack + */ +public class Doc { + + + final private static Logger LOGGER = Logger.getLogger(Doc.class.getName()); + + + /** + * Static utility library, do not instantiate. + */ + private Doc() { + } + + + /** + * Returns the text of the .doc file with the given file name. + * + * @param docFilename the name of the file whose text to return + * @return the text of that file + * @throws IOException if an IO error occurs + */ + public static SeekReader getText(String docFilename) throws IOException { + return getText(new File(docFilename)); + } + + + /** + * Returns the text of the given .doc file. + * + * @param doc the .doc file whose text to return + * @return the text of that file + * @throws IOException if an IO error occurs + */ + public static SeekReader getText(File doc) throws IOException { + RandomAccessFile raf = new RandomAccessFile(doc, "r"); + RandomAccessInputStream rais = new RandomAccessInputStream(raf); + return getText(rais); + } + + + /** + * Returns the text of the given .doc file. + * + * @param doc the .doc file whose text to return + * @return the text of that file + * @throws IOException if an IO error occurs + */ + public static SeekReader getText(SeekInputStream doc) throws IOException { + BlockFileSystem bfs = new DefaultBlockFileSystem(doc, 16); + return getText(bfs, 20); + } + + + /** + * Returns the text for the given .doc file. The given cacheSize refers + * to the number of the .doc file's piece table entries to cache. Most + * .doc files only have 1 piece table entry; however, a "fast-saved" + * .doc file might have several. A cacheSize of 20 should be ample for + * most .doc files in the world. Since piece table entries are small -- + * only 12 bytes each -- caching them prevents many otherwise necessary + * file pointer repositionings. + * + * @param wordDoc the .doc file as a BlockFileSystem + * @param cacheSize the number of piece table entries to cache + * @return a reader that will return the text in the file + * @throws IOException if an IO error occurs + */ + public static SeekReader getText(BlockFileSystem wordDoc, int cacheSize) + throws IOException { + List entries = wordDoc.getRoot().list(); + Entry main = find(entries, "WordDocument"); + SeekInputStream mainStream = main.open(); + + mainStream.position(10); + int flags = Endian.littleChar(mainStream); + boolean complex = (flags & 0x0004) == 0x0004; + boolean tableOne = (flags & 0x0200) == 0x0200; + String tableName = tableOne ? "1Table" : "0Table"; + Entry table = find(entries, tableName); + if (LOGGER.isLoggable(Level.FINEST)) { + LOGGER.finest("Main entry: " + main); + LOGGER.finest("Table entry: " + table); + } + SeekInputStream tableStream = table.open(); + + mainStream.position(24); + int fcMin = Endian.littleInt(mainStream); + int fcMax = Endian.littleInt(mainStream); + + mainStream.position(76); + int cppText = Endian.littleInt(mainStream); + + mainStream.position(418); + int fcClx = Endian.littleInt(mainStream); + int fcSz = Endian.littleInt(mainStream); + + if (LOGGER.isLoggable(Level.FINE)) { + LOGGER.fine("fcMin: " + fcMin); + LOGGER.fine("fcMax: " + fcMax); + LOGGER.fine("FcClx: " + fcClx); + LOGGER.fine("szClx: " + fcSz); + LOGGER.fine("complex: " + complex); + LOGGER.fine("cppText: " + cppText); + } + PieceTable pt = new PieceTable(tableStream, fcClx, fcMax - fcMin, cacheSize); + return new PieceReader(pt, mainStream); + } + + + private static Entry find(List entries, String name) { + for (Entry e: entries) { + if (e.getName().equals(name)) { + return e; + } + } + return null; + } + +} diff --git a/commons/src/main/java/org/archive/util/ms/Entry.java b/commons/src/main/java/org/archive/util/ms/Entry.java new file mode 100644 index 00000000..27e854b0 --- /dev/null +++ b/commons/src/main/java/org/archive/util/ms/Entry.java @@ -0,0 +1,47 @@ +/* Entry +* +* Created on September 12, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util.ms; + +import java.io.IOException; +import java.util.List; + +import org.archive.io.SeekInputStream; + + +public interface Entry { + + + enum EntryType { ROOT, FILE, DIRECTORY }; + + String getName(); + int getIndex(); + + Entry getPrevious() throws IOException; + Entry getNext() throws IOException; + Entry getChild() throws IOException; + EntryType getType() throws IOException; + List list() throws IOException; + SeekInputStream open() throws IOException; + + +} diff --git a/commons/src/main/java/org/archive/util/ms/HeaderBlock.java b/commons/src/main/java/org/archive/util/ms/HeaderBlock.java new file mode 100644 index 00000000..c5d4ba6c --- /dev/null +++ b/commons/src/main/java/org/archive/util/ms/HeaderBlock.java @@ -0,0 +1,97 @@ +/* HeaderBlock +* +* Created on September 12, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util.ms; + + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + + +class HeaderBlock { + + + private ByteBuffer buffer; + + + public HeaderBlock(ByteBuffer buffer) { + // FIXME: Read the fields we're interested in directly from stream + this.buffer = buffer; + buffer.order(ByteOrder.LITTLE_ENDIAN); + } + + + public long getFileType() { + return buffer.getLong(0); + } + + + public int getBATCount() { + return buffer.getInt(0x2C); + } + + + public int getEntriesStart() { + return buffer.getInt(0x30); + } + + + public int getSmallBATStart() { + return buffer.getInt(0x3C); + } + + + public int getSmallBATCount() { + return buffer.getInt(0x40); + } + + + public int getExtendedBATStart() { + return buffer.getInt(0x44); + } + + + public int getExtendedBATCount() { + return buffer.getInt(0x48); + } + + + public int getBATBlockNumber(int block) { + assert block < 110; + return buffer.getInt(0x4C + block * 4); + } + + + public String toString() { + StringBuilder sb = new StringBuilder("HeaderBlock{"); + sb.append("fileType=" + getFileType()); + sb.append(" propertiesStart=" + getEntriesStart()); + sb.append(" batCount=" + getBATCount()); + sb.append(" extendedBATStart=" + getExtendedBATStart()); + sb.append(" extendedBATCount=" + getExtendedBATCount()); + sb.append(" smallBATStart=" + getSmallBATStart()); + sb.append(" smallBATCount=" + getSmallBATCount()); + sb.append("}"); + return sb.toString(); + } + +} diff --git a/commons/src/main/java/org/archive/util/ms/Piece.java b/commons/src/main/java/org/archive/util/ms/Piece.java new file mode 100644 index 00000000..edc35c8b --- /dev/null +++ b/commons/src/main/java/org/archive/util/ms/Piece.java @@ -0,0 +1,86 @@ +/* Piece +* +* Created on September 12, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util.ms; + +public class Piece { + + private boolean unicode; + private int charPosStart; + private int charPosLimit; + private int filePos; + + + public Piece(int filePos, int start, int end, boolean unicode) { + this.filePos = filePos; + this.charPosStart = start; + this.charPosLimit = end; + this.unicode = unicode; + } + + + /** + * + * @return + */ + public int getFilePos() { + return filePos; + } + + + /** + * + * @return + */ + public int getCharPosLimit() { + return charPosLimit; + } + + + public int getCharPosStart() { + return charPosStart; + } + + /** + * + * @return + */ + public boolean isUnicode() { + return unicode; + } + + + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("Piece{filePos=").append(filePos); + sb.append(" start=").append(charPosStart); + sb.append(" end=").append(charPosLimit); + sb.append(" unicode=").append(unicode); + sb.append("}"); + return sb.toString(); + } + + + public boolean contains(int charPos) { + return (charPos >= charPosStart) && (charPos < charPosLimit); + } +} diff --git a/commons/src/main/java/org/archive/util/ms/PieceReader.java b/commons/src/main/java/org/archive/util/ms/PieceReader.java new file mode 100644 index 00000000..e747aad8 --- /dev/null +++ b/commons/src/main/java/org/archive/util/ms/PieceReader.java @@ -0,0 +1,137 @@ +/* PieceReader +* +* Created on September 12, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util.ms; + +import java.io.IOException; + +import org.archive.io.Endian; +import org.archive.io.SeekInputStream; +import org.archive.io.SeekReader; + + +class PieceReader extends SeekReader { + + + private PieceTable table; + private SeekInputStream doc; + + private boolean unicode; + private int charPos; + private int limit; + + + public PieceReader(PieceTable table, SeekInputStream doc) + throws IOException { + this.table = table; + this.doc = doc; + charPos = 0; + limit = -1; + } + + + private void seekIfNecessary() throws IOException { + if (doc == null) { + throw new IOException("Stream closed."); + } + if (charPos >= table.getMaxCharPos()) { + return; + } + if (charPos < limit) { + return; + } + Piece piece = table.next(); + unicode = piece.isUnicode(); + limit = piece.getCharPosLimit(); + doc.position(piece.getFilePos()); + } + + + public int read() throws IOException { + seekIfNecessary(); + if (doc == null) { + throw new IOException("Stream closed."); + } + if (charPos >= table.getMaxCharPos()) { + return -1; + } + + int ch; + if (unicode) { + ch = Endian.littleChar(doc); + } else { + ch = Cp1252.decode(doc.read()); + } + charPos++; + return ch; + } + + + public int read(char[] buf, int ofs, int len) throws IOException { + // FIXME: Think of a faster implementation that will work with + // both unicode and non-unicode. + seekIfNecessary(); + if (doc == null) { + throw new IOException("Stream closed."); + } + if (charPos >= table.getMaxCharPos()) { + return 0; + } + for (int i = 0; i < len; i++) { + int ch = read(); + if (ch < 0) { + return i; + } + buf[ofs + i] = (char)ch; + } + return len; + } + + + public void close() throws IOException { + doc.close(); + table = null; + } + + + public long position() throws IOException { + return charPos; + } + + + public void position(long p) throws IOException { + if (p > Integer.MAX_VALUE) { + throw new IOException("File too large."); + } + int charPos = (int)p; + Piece piece = table.pieceFor(charPos); + if (piece == null) { + throw new IOException("Illegal position: " + p); + } + unicode = piece.isUnicode(); + limit = piece.getCharPosLimit(); + + int ofs = charPos - piece.getCharPosStart(); + this.charPos = charPos; + doc.position(piece.getFilePos() + ofs); + } +} diff --git a/commons/src/main/java/org/archive/util/ms/PieceTable.java b/commons/src/main/java/org/archive/util/ms/PieceTable.java new file mode 100644 index 00000000..398d7912 --- /dev/null +++ b/commons/src/main/java/org/archive/util/ms/PieceTable.java @@ -0,0 +1,284 @@ +/* PieceTable +* +* Created on September 12, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util.ms; + +import java.io.IOException; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.archive.io.BufferedSeekInputStream; +import org.archive.io.Endian; +import org.archive.io.OriginSeekInputStream; +import org.archive.io.SafeSeekInputStream; +import org.archive.io.SeekInputStream; + + +/** + * The piece table of a .doc file. + * + *

The piece table maps logical character positions of a document's text + * stream to actual file stream positions. The piece table is stored as two + * parallel arrays. The first array contains 32-bit integers representing + * the logical character positions. The second array contains 64-bit data + * structures that are mostly mysterious to me, except that they contain a + * 32-bit subfile offset. The second array is stored immediately after the + * first array. I call the first array the charPos array and the + * second array the filePos array. + * + *

The arrays are preceded by a special tag byte (2), followed by the + * combined size of both arrays in bytes. The number of piece table entries + * must be deduced from this byte size. + * + *

Because of this bizarre structure, caching piece table entries is + * something of a challenge. A single piece table entry is actually located + * in two different file locations. If there are many piece table entries, + * then the charPos and filePos information may be separated by many bytes, + * potentially crossing block boundaries. The approach I took was to use + * two different buffered streams. Up to n charPos offsets and n filePos + * structures can be buffered in the two streams, preventing any file seeking + * from occurring when looking up piece information. (File seeking must + * still occur to jump from one piece to the next.) + * + *

Note that the vast majority of .doc files in the world will have exactly + * 1 piece table entry, representing the complete text of the document. Only + * those documents that were "fast-saved" should have multiple pieces. + * + *

Finally, the text contained in a .doc file can either contain 16-bit + * unicode characters (charset UTF-16LE) or 8-bit CP1252 characters. One + * .doc file can contain both kinds of pieces. Whether or not a piece is + * Cp1252 is stored as a flag in the filePos value, bizarrely enough. If + * the flag is set, then the actual file position is the filePos with the + * flag cleared, then divided by 2. + * + * @author pjack + */ +class PieceTable { + + final static Logger LOGGER + = Logger.getLogger(PieceTable.class.getName()); + + /** The bit that indicates if a piece uses Cp1252 or unicode. */ + final static int CP1252_INDICATOR = 1 << 30; + + /** The mask to use to clear the Cp1252 flag bit. */ + final static int CP1252_MASK = ~(3 << 30); + + /** The total number of pieces in the table. */ + private int count; + + /** The total number of characters in the text stream. */ + private int maxCharPos; + + /** The index of the current piece. */ + private int current; + + /** The most recently returned piece from this table. */ + private Piece currentPiece; + + + /** The buffered stream that provides character position information. */ + private SeekInputStream charPos; + + /** The buffered stream that provides file pointer information. */ + private SeekInputStream filePos; + + + /** + * Constructor. + * + * @param tableStream the stream containing the piece table + * @param offset the starting offset of the piece table + * @param maxCharPos the total number of characters in the document + * @param cachedRecords the number of piece table entries to cache + * @throws IOException if an IO error occurs + */ + public PieceTable(SeekInputStream tableStream, int offset, + int maxCharPos, int cachedRecords) throws IOException { + tableStream.position(offset); + skipProperties(tableStream); + int sizeInBytes = Endian.littleInt(tableStream); + this.count = (sizeInBytes - 4) / 12; + cachedRecords = Math.min(cachedRecords, count); + long tp = tableStream.position() + 4; + long charPosStart = tp; + long filePosStart = tp + count * 4 + 2; + + this.filePos = wrap(tableStream, filePosStart, cachedRecords * 8); + this.charPos = wrap(tableStream, charPosStart, cachedRecords * 4); + this.maxCharPos = maxCharPos; + + if (LOGGER.isLoggable(Level.FINEST)) { + LOGGER.finest("Size in bytes: " + sizeInBytes); + LOGGER.finest("Piece table count: " + count); + for (Piece piece = next(); piece != null; piece = next()) { + LOGGER.finest("#" + current + ": " + piece.toString()); + } + current = 0; + } + } + + + /** + * Wraps the raw table stream. This is used to create the charPos and + * filePos streams. The streams that this method returns are "safe", + * meaning that the charPos and filePos position() fields never clobber + * each other. They are buffered, meaning that up to n elements + * can be read before the disk is accessed again. And they are "origined", + * meaning result.position(0) actually positions the stream at the + * beginning of the piece table array, not the beginning of the file. + * + * @param input the stream to wrap + * @param pos the origin for the returned stream + * @param cache the number of bytes for the returned stream to buffer + * @return the wrapped stream + * @throws IOException if an IO error occurs + */ + private SeekInputStream wrap(SeekInputStream input, long pos, int cache) + throws IOException { + input.position(pos); + SeekInputStream r = new SafeSeekInputStream(input); + r = new OriginSeekInputStream(r, pos); + r = new BufferedSeekInputStream(r, cache); + return r; + } + + + /** + * Skips over any property information that may precede a piece table. + * These property structures contain stylesheet information that applies + * to the piece table. Since we're only interested in the text itself, + * we just ignore this property stuff. (I suppose a third buffered + * stream could be used to add style information to {@link Piece}, but + * we don't need it.) + * + * @param input the input stream containing the piece table + * @throws IOException if an IO error occurs + */ + private static void skipProperties(SeekInputStream input) throws IOException { + int tag = input.read(); + while (tag == 1) { + int size = Endian.littleChar(input); + while (size > 0) { + size -= input.skip(size); + } + tag = input.read(); + } + if (tag != 2) { + throw new IllegalStateException(); + } + } + + + /** + * Returns the maximum character position. Put another way, returns the + * total number of characters in the document. + * + * @return the maximum character position + */ + public int getMaxCharPos() { + return maxCharPos; + } + + + /** + * Returns the next piece in the piece table. + * + * @return the next piece in the piece table, or null if there is no + * next piece + * @throws IOException if an IO error occurs + */ + public Piece next() throws IOException { + if (current >= count) { + currentPiece = null; + return null; + } + + int cp; + if (current == count - 1) { + cp = maxCharPos; + } else { + charPos.position(current * 4); + cp = Endian.littleInt(charPos); + } + filePos.position(current * 8); + int encoded = Endian.littleInt(filePos); + + if (LOGGER.isLoggable(Level.FINEST)) { + StringBuffer sb = new StringBuffer(Integer.toBinaryString(encoded)); + while (sb.length() < 32) { + sb.insert(0, '0'); + } + LOGGER.finest("Encoded offset: " + sb.toString()); + } + + current++; + + int start; + if (currentPiece == null) { + start = 0; + } else { + start = currentPiece.getCharPosLimit(); + } + if ((encoded & CP1252_INDICATOR) == 0) { + Piece piece = new Piece(encoded, start, cp, true); + currentPiece = piece; + return piece; + } else { + int filePos = (encoded & CP1252_MASK) / 2; + Piece piece = new Piece(filePos, start, cp, false); + currentPiece = piece; + return piece; + } + } + + + /** + * Returns the piece containing the given character position. + * + * @param charPos the character position whose piece to return + * @return that piece, or null if no such piece exists (if charPos + * is greater than getMaxCharPos()) + * @throws IOException if an IO error occurs + */ + public Piece pieceFor(int charPos) throws IOException { + if (currentPiece.contains(charPos)) { + return currentPiece; + } + + // FIXME: Use binary search to find piece index + + current = 0; + currentPiece = null; + next(); + + while (currentPiece != null) { + if (currentPiece.contains(charPos)) { + return currentPiece; + } + next(); + } + + return null; + } + +} diff --git a/commons/src/main/java/org/archive/util/ms/package.html b/commons/src/main/java/org/archive/util/ms/package.html new file mode 100644 index 00000000..82b3bb38 --- /dev/null +++ b/commons/src/main/java/org/archive/util/ms/package.html @@ -0,0 +1,114 @@ + + +Memory-efficient reading of .doc files. To extract the text from a .doc +file, use {@link org.archive.util.ms.Doc#getText(SeekInputStream)}. That's +basically the whole API. The other classes are necessary to make that +method work, and you can probably ignore them. + +

Implementation/Format Details

+ +

These APIs differ from the POI API provided by Apache in that POI wants to +load complete documents into memory. Though POI does provide an "event-driven" +API that is memory efficient, that API cannot be used to scan text across block +or piece boundaries. + +

This package provides a stream-based API for extracting the text of +a .doc file. At this time, the package does not provide a way to extract +style attributes, embedded images, subdocuments, change tracking information, +and so on. + +

There are two layers of abstraction between the contents of a .doc +file and reality. The first layer is the Block File System, and +the second layer is the piece table. + +

The Block File System

+ +

All .doc files are secretly file systems, like a .iso file, but insane. +A good overview of how this file system is arranged inside the file is +available at +the Jarkarta POIFS system. + +

Subfiles and directories in a block file system are represented via the +{@link org.archive.util.ms.Entry} interface. The root directory can be +obtained via the {@link org.archive.util.ms.BlockFileSystem#getRoot()} +method. From there, the child entries can be discovered. + +

The file system divides its subfiles into 512-byte blocks. Those blocks +are not necessarily stored in a linear order; blocks from different subfiles +may be interspersed with each other. The +{@link org.archive.util.ms.Entry#open()} method returns an input stream that +provides a continuous view of a subfile's contents. It does so by moving +the file pointer of the .doc file behind the scenes. + +

It's important to keep in mind that any given read on a stream produced +by a BlockFileSystem may involve: + +

    +
  1. Moving the file pointer to the start of the file to look up the main +block allocation table.
    2. +
  2. Navigation the file pointer through various allocation structures located +throughout the file.
    3. +
  3. Finally repositioning the file pointer at the start of the next block +to be read.
    4. +
+ +

So, this package lowers memory consumption at the expense of greater IO +activity. A future version of this package will use internal caches to +minimize IO activity, providing tunable trade-offs between memory and IO. + +

The Piece Table

+ +

The second layer of abstraction between you and the contents of a .doc file +is the piece table. Some .doc files are produced using a "fast-save" feature +that only writes recent changes to the end of the file. In this case, the +text of the document may be fragmented within the document stream itself. +Note that this fragmentation is in addition to the block fragmentation +described above. + +

A .doc file contains several subfiles within its filesystem. The two +that are important for extracting text are named WordDocument +and 0Table. The WordDocument subfile contains the +text of the document. The 0Table subfile contains supporting +information, including the piece table. + +

The piece table is a simple map from logical character position to actual +subfile stream position. Additionally, each piece table entry describes whether +or not the piece stores text using 16-bit Unicode, or using 8-bit ANSI +codes. One .doc file can contain both Unicode and ANSI text. A consequence +of this is that every .doc file has a piece table, even those that +were not "fast-saved". + +

The reader returned by +{@link org.achive.util.ms.Doc#getText(SeekInputStream)} consults the piece +table to determine where in the WordDocument subfile the next piece of text +is located. It also uses the piece table to determine how bytes should be +converted to Unicode characters. + +

Note, however, that any read from such a reader may involve: + +

    +
  1. Moving the file pointer to the piece table.
    2. +
  2. Searching the piece table index for the next piece, which may +involve moving the file pointer many times.
    3. +
  3. Moving the file pointer to that piece's description in the piece table.
    4. +
  4. Moving the file pointer to the start of the piece indicated by the +description.
    5. +
+ +Since the "file pointer" in this context is the file pointer of the +subfile, each move described above may additionally involve: + +
    +
  1. Moving the file pointer to the piece table.
    2. +
  2. Searching the piece table index for the next piece, which may +involve moving the file pointer many times.
    3. +
  3. Moving the file pointer to that piece's description in the piece table.
    4. +
  4. Moving the file pointer to the start of the piece indicated by the +description.
    5. +
+ +A future implementation will provide an intelligent cache of the piece table, +which will hopefully reduce the IO activity required. + + + diff --git a/commons/src/main/java/org/json/JSONArray.java b/commons/src/main/java/org/json/JSONArray.java new file mode 100644 index 00000000..b15d832b --- /dev/null +++ b/commons/src/main/java/org/json/JSONArray.java @@ -0,0 +1,890 @@ +package org.json; + +/* +Copyright (c) 2002 JSON.org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +The Software shall be used for Good, not Evil. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +import java.io.IOException; +import java.io.Writer; +import java.lang.reflect.Array; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; + +/** + * A JSONArray is an ordered sequence of values. Its external text form is a + * string wrapped in square brackets with commas separating the values. The + * internal form is an object having get and opt + * methods for accessing the values by index, and put methods for + * adding or replacing values. The values can be any of these types: + * Boolean, JSONArray, JSONObject, + * Number, String, or the + * JSONObject.NULL object. + *

+ * The constructor can convert a JSON text into a Java object. The + * toString method converts to JSON text. + *

+ * A get method returns a value if one can be found, and throws an + * exception if one cannot be found. An opt method returns a + * default value instead of throwing an exception, and so is useful for + * obtaining optional values. + *

+ * The generic get() and opt() methods return an + * object which you can cast or query for type. There are also typed + * get and opt methods that do type checking and type + * coersion for you. + *

+ * The texts produced by the toString methods strictly conform to + * JSON syntax rules. The constructors are more forgiving in the texts they will + * accept: + *

    + *
  • An extra , (comma) may appear just + * before the closing bracket.
    • + *
  • The null value will be inserted when there + * is , (comma) elision.
    • + *
  • Strings may be quoted with ' (single + * quote).
    • + *
  • Strings do not need to be quoted at all if they do not begin with a quote + * or single quote, and if they do not contain leading or trailing spaces, + * and if they do not contain any of these characters: + * { } [ ] / \ : , = ; # and if they do not look like numbers + * and if they are not the reserved words true, + * false, or null.
    • + *
  • Values can be separated by ; (semicolon) as + * well as by , (comma).
    • + *
  • Numbers may have the 0- (octal) or + * 0x- (hex) prefix.
    • + *
  • Comments written in the slashshlash, slashstar, and hash conventions + * will be ignored.
    • + *
+ + * @author JSON.org + * @version 2 + */ +@SuppressWarnings("unchecked") +public class JSONArray { + + + /** + * The arrayList where the JSONArray's properties are kept. + */ + private ArrayList myArrayList; + + + /** + * Construct an empty JSONArray. + */ + public JSONArray() { + this.myArrayList = new ArrayList(); + } + + /** + * Construct a JSONArray from a JSONTokener. + * @param x A JSONTokener + * @throws JSONException If there is a syntax error. + */ + public JSONArray(JSONTokener x) throws JSONException { + this(); + if (x.nextClean() != '[') { + throw x.syntaxError("A JSONArray text must start with '['"); + } + if (x.nextClean() == ']') { + return; + } + x.back(); + for (;;) { + if (x.nextClean() == ',') { + x.back(); + this.myArrayList.add(null); + } else { + x.back(); + this.myArrayList.add(x.nextValue()); + } + switch (x.nextClean()) { + case ';': + case ',': + if (x.nextClean() == ']') { + return; + } + x.back(); + break; + case ']': + return; + default: + throw x.syntaxError("Expected a ',' or ']'"); + } + } + } + + + /** + * Construct a JSONArray from a source sJSON text. + * @param string A string that begins with + * [ (left bracket) + * and ends with ] (right bracket). + * @throws JSONException If there is a syntax error. + */ + public JSONArray(String string) throws JSONException { + this(new JSONTokener(string)); + } + + + /** + * Construct a JSONArray from a Collection. + * @param collection A Collection. + */ + public JSONArray(Collection collection) { + this.myArrayList = (collection == null) ? + new ArrayList() : + new ArrayList(collection); + } + + + /** + * Construct a JSONArray from an array + * @throws JSONException If not an array. + */ + public JSONArray(Object array) throws JSONException { + this(); + if (array.getClass().isArray()) { + int length = Array.getLength(array); + for (int i = 0; i < length; i += 1) { + this.put(Array.get(array, i)); + } + } else { + throw new JSONException("JSONArray initial value should be a string or collection or array."); + } + } + + + /** + * Get the object value associated with an index. + * @param index + * The index must be between 0 and length() - 1. + * @return An object value. + * @throws JSONException If there is no value for the index. + */ + public Object get(int index) throws JSONException { + Object o = opt(index); + if (o == null) { + throw new JSONException("JSONArray[" + index + "] not found."); + } + return o; + } + + + /** + * Get the boolean value associated with an index. + * The string values "true" and "false" are converted to boolean. + * + * @param index The index must be between 0 and length() - 1. + * @return The truth. + * @throws JSONException If there is no value for the index or if the + * value is not convertable to boolean. + */ + public boolean getBoolean(int index) throws JSONException { + Object o = get(index); + if (o.equals(Boolean.FALSE) || + (o instanceof String && + ((String)o).equalsIgnoreCase("false"))) { + return false; + } else if (o.equals(Boolean.TRUE) || + (o instanceof String && + ((String)o).equalsIgnoreCase("true"))) { + return true; + } + throw new JSONException("JSONArray[" + index + "] is not a Boolean."); + } + + + /** + * Get the double value associated with an index. + * + * @param index The index must be between 0 and length() - 1. + * @return The value. + * @throws JSONException If the key is not found or if the value cannot + * be converted to a number. + */ + public double getDouble(int index) throws JSONException { + Object o = get(index); + try { + return o instanceof Number ? + ((Number)o).doubleValue() : + Double.valueOf((String)o).doubleValue(); + } catch (Exception e) { + throw new JSONException("JSONArray[" + index + + "] is not a number."); + } + } + + + /** + * Get the int value associated with an index. + * + * @param index The index must be between 0 and length() - 1. + * @return The value. + * @throws JSONException If the key is not found or if the value cannot + * be converted to a number. + * if the value cannot be converted to a number. + */ + public int getInt(int index) throws JSONException { + Object o = get(index); + return o instanceof Number ? + ((Number)o).intValue() : (int)getDouble(index); + } + + + /** + * Get the JSONArray associated with an index. + * @param index The index must be between 0 and length() - 1. + * @return A JSONArray value. + * @throws JSONException If there is no value for the index. or if the + * value is not a JSONArray + */ + public JSONArray getJSONArray(int index) throws JSONException { + Object o = get(index); + if (o instanceof JSONArray) { + return (JSONArray)o; + } + throw new JSONException("JSONArray[" + index + + "] is not a JSONArray."); + } + + + /** + * Get the JSONObject associated with an index. + * @param index subscript + * @return A JSONObject value. + * @throws JSONException If there is no value for the index or if the + * value is not a JSONObject + */ + public JSONObject getJSONObject(int index) throws JSONException { + Object o = get(index); + if (o instanceof JSONObject) { + return (JSONObject)o; + } + throw new JSONException("JSONArray[" + index + + "] is not a JSONObject."); + } + + + /** + * Get the long value associated with an index. + * + * @param index The index must be between 0 and length() - 1. + * @return The value. + * @throws JSONException If the key is not found or if the value cannot + * be converted to a number. + */ + public long getLong(int index) throws JSONException { + Object o = get(index); + return o instanceof Number ? + ((Number)o).longValue() : (long)getDouble(index); + } + + + /** + * Get the string associated with an index. + * @param index The index must be between 0 and length() - 1. + * @return A string value. + * @throws JSONException If there is no value for the index. + */ + public String getString(int index) throws JSONException { + return get(index).toString(); + } + + + /** + * Determine if the value is null. + * @param index The index must be between 0 and length() - 1. + * @return true if the value at the index is null, or if there is no value. + */ + public boolean isNull(int index) { + return JSONObject.NULL.equals(opt(index)); + } + + + /** + * Make a string from the contents of this JSONArray. The + * separator string is inserted between each element. + * Warning: This method assumes that the data structure is acyclical. + * @param separator A string that will be inserted between the elements. + * @return a string. + * @throws JSONException If the array contains an invalid number. + */ + public String join(String separator) throws JSONException { + int len = length(); + StringBuffer sb = new StringBuffer(); + + for (int i = 0; i < len; i += 1) { + if (i > 0) { + sb.append(separator); + } + sb.append(JSONObject.valueToString(this.myArrayList.get(i))); + } + return sb.toString(); + } + + + /** + * Get the number of elements in the JSONArray, included nulls. + * + * @return The length (or size). + */ + public int length() { + return this.myArrayList.size(); + } + + + /** + * Get the optional object value associated with an index. + * @param index The index must be between 0 and length() - 1. + * @return An object value, or null if there is no + * object at that index. + */ + public Object opt(int index) { + return (index < 0 || index >= length()) ? + null : this.myArrayList.get(index); + } + + + /** + * Get the optional boolean value associated with an index. + * It returns false if there is no value at that index, + * or if the value is not Boolean.TRUE or the String "true". + * + * @param index The index must be between 0 and length() - 1. + * @return The truth. + */ + public boolean optBoolean(int index) { + return optBoolean(index, false); + } + + + /** + * Get the optional boolean value associated with an index. + * It returns the defaultValue if there is no value at that index or if + * it is not a Boolean or the String "true" or "false" (case insensitive). + * + * @param index The index must be between 0 and length() - 1. + * @param defaultValue A boolean default. + * @return The truth. + */ + public boolean optBoolean(int index, boolean defaultValue) { + try { + return getBoolean(index); + } catch (Exception e) { + return defaultValue; + } + } + + + /** + * Get the optional double value associated with an index. + * NaN is returned if there is no value for the index, + * or if the value is not a number and cannot be converted to a number. + * + * @param index The index must be between 0 and length() - 1. + * @return The value. + */ + public double optDouble(int index) { + return optDouble(index, Double.NaN); + } + + + /** + * Get the optional double value associated with an index. + * The defaultValue is returned if there is no value for the index, + * or if the value is not a number and cannot be converted to a number. + * + * @param index subscript + * @param defaultValue The default value. + * @return The value. + */ + public double optDouble(int index, double defaultValue) { + try { + return getDouble(index); + } catch (Exception e) { + return defaultValue; + } + } + + + /** + * Get the optional int value associated with an index. + * Zero is returned if there is no value for the index, + * or if the value is not a number and cannot be converted to a number. + * + * @param index The index must be between 0 and length() - 1. + * @return The value. + */ + public int optInt(int index) { + return optInt(index, 0); + } + + + /** + * Get the optional int value associated with an index. + * The defaultValue is returned if there is no value for the index, + * or if the value is not a number and cannot be converted to a number. + * @param index The index must be between 0 and length() - 1. + * @param defaultValue The default value. + * @return The value. + */ + public int optInt(int index, int defaultValue) { + try { + return getInt(index); + } catch (Exception e) { + return defaultValue; + } + } + + + /** + * Get the optional JSONArray associated with an index. + * @param index subscript + * @return A JSONArray value, or null if the index has no value, + * or if the value is not a JSONArray. + */ + public JSONArray optJSONArray(int index) { + Object o = opt(index); + return o instanceof JSONArray ? (JSONArray)o : null; + } + + + /** + * Get the optional JSONObject associated with an index. + * Null is returned if the key is not found, or null if the index has + * no value, or if the value is not a JSONObject. + * + * @param index The index must be between 0 and length() - 1. + * @return A JSONObject value. + */ + public JSONObject optJSONObject(int index) { + Object o = opt(index); + return o instanceof JSONObject ? (JSONObject)o : null; + } + + + /** + * Get the optional long value associated with an index. + * Zero is returned if there is no value for the index, + * or if the value is not a number and cannot be converted to a number. + * + * @param index The index must be between 0 and length() - 1. + * @return The value. + */ + public long optLong(int index) { + return optLong(index, 0); + } + + + /** + * Get the optional long value associated with an index. + * The defaultValue is returned if there is no value for the index, + * or if the value is not a number and cannot be converted to a number. + * @param index The index must be between 0 and length() - 1. + * @param defaultValue The default value. + * @return The value. + */ + public long optLong(int index, long defaultValue) { + try { + return getLong(index); + } catch (Exception e) { + return defaultValue; + } + } + + + /** + * Get the optional string value associated with an index. It returns an + * empty string if there is no value at that index. If the value + * is not a string and is not null, then it is coverted to a string. + * + * @param index The index must be between 0 and length() - 1. + * @return A String value. + */ + public String optString(int index) { + return optString(index, ""); + } + + + /** + * Get the optional string associated with an index. + * The defaultValue is returned if the key is not found. + * + * @param index The index must be between 0 and length() - 1. + * @param defaultValue The default value. + * @return A String value. + */ + public String optString(int index, String defaultValue) { + Object o = opt(index); + return o != null ? o.toString() : defaultValue; + } + + + /** + * Append a boolean value. This increases the array's length by one. + * + * @param value A boolean value. + * @return this. + */ + public JSONArray put(boolean value) { + put(value ? Boolean.TRUE : Boolean.FALSE); + return this; + } + + + /** + * Put a value in the JSONArray, where the value will be a + * JSONArray which is produced from a Collection. + * @param value A Collection value. + * @return this. + */ + public JSONArray put(Collection value) { + put(new JSONArray(value)); + return this; + } + + + /** + * Append a double value. This increases the array's length by one. + * + * @param value A double value. + * @throws JSONException if the value is not finite. + * @return this. + */ + public JSONArray put(double value) throws JSONException { + Double d = new Double(value); + JSONObject.testValidity(d); + put(d); + return this; + } + + + /** + * Append an int value. This increases the array's length by one. + * + * @param value An int value. + * @return this. + */ + public JSONArray put(int value) { + put(new Integer(value)); + return this; + } + + + /** + * Append an long value. This increases the array's length by one. + * + * @param value A long value. + * @return this. + */ + public JSONArray put(long value) { + put(new Long(value)); + return this; + } + + + /** + * Put a value in the JSONArray, where the value will be a + * JSONObject which is produced from a Map. + * @param value A Map value. + * @return this. + */ + public JSONArray put(Map value) { + put(new JSONObject(value)); + return this; + } + + + /** + * Append an object value. This increases the array's length by one. + * @param value An object value. The value should be a + * Boolean, Double, Integer, JSONArray, JSONObject, Long, or String, or the + * JSONObject.NULL object. + * @return this. + */ + public JSONArray put(Object value) { + this.myArrayList.add(value); + return this; + } + + + /** + * Put or replace a boolean value in the JSONArray. If the index is greater + * than the length of the JSONArray, then null elements will be added as + * necessary to pad it out. + * @param index The subscript. + * @param value A boolean value. + * @return this. + * @throws JSONException If the index is negative. + */ + public JSONArray put(int index, boolean value) throws JSONException { + put(index, value ? Boolean.TRUE : Boolean.FALSE); + return this; + } + + + /** + * Put a value in the JSONArray, where the value will be a + * JSONArray which is produced from a Collection. + * @param index The subscript. + * @param value A Collection value. + * @return this. + * @throws JSONException If the index is negative or if the value is + * not finite. + */ + public JSONArray put(int index, Collection value) throws JSONException { + put(index, new JSONArray(value)); + return this; + } + + + /** + * Put or replace a double value. If the index is greater than the length of + * the JSONArray, then null elements will be added as necessary to pad + * it out. + * @param index The subscript. + * @param value A double value. + * @return this. + * @throws JSONException If the index is negative or if the value is + * not finite. + */ + public JSONArray put(int index, double value) throws JSONException { + put(index, new Double(value)); + return this; + } + + + /** + * Put or replace an int value. If the index is greater than the length of + * the JSONArray, then null elements will be added as necessary to pad + * it out. + * @param index The subscript. + * @param value An int value. + * @return this. + * @throws JSONException If the index is negative. + */ + public JSONArray put(int index, int value) throws JSONException { + put(index, new Integer(value)); + return this; + } + + + /** + * Put or replace a long value. If the index is greater than the length of + * the JSONArray, then null elements will be added as necessary to pad + * it out. + * @param index The subscript. + * @param value A long value. + * @return this. + * @throws JSONException If the index is negative. + */ + public JSONArray put(int index, long value) throws JSONException { + put(index, new Long(value)); + return this; + } + + + /** + * Put a value in the JSONArray, where the value will be a + * JSONObject which is produced from a Map. + * @param index The subscript. + * @param value The Map value. + * @return this. + * @throws JSONException If the index is negative or if the the value is + * an invalid number. + */ + public JSONArray put(int index, Map value) throws JSONException { + put(index, new JSONObject(value)); + return this; + } + + + /** + * Put or replace an object value in the JSONArray. If the index is greater + * than the length of the JSONArray, then null elements will be added as + * necessary to pad it out. + * @param index The subscript. + * @param value The value to put into the array. The value should be a + * Boolean, Double, Integer, JSONArray, JSONObject, Long, or String, or the + * JSONObject.NULL object. + * @return this. + * @throws JSONException If the index is negative or if the the value is + * an invalid number. + */ + public JSONArray put(int index, Object value) throws JSONException { + JSONObject.testValidity(value); + if (index < 0) { + throw new JSONException("JSONArray[" + index + "] not found."); + } + if (index < length()) { + this.myArrayList.set(index, value); + } else { + while (index != length()) { + put(JSONObject.NULL); + } + put(value); + } + return this; + } + + + /** + * Produce a JSONObject by combining a JSONArray of names with the values + * of this JSONArray. + * @param names A JSONArray containing a list of key strings. These will be + * paired with the values. + * @return A JSONObject, or null if there are no names or if this JSONArray + * has no values. + * @throws JSONException If any of the names are null. + */ + public JSONObject toJSONObject(JSONArray names) throws JSONException { + if (names == null || names.length() == 0 || length() == 0) { + return null; + } + JSONObject jo = new JSONObject(); + for (int i = 0; i < names.length(); i += 1) { + jo.put(names.getString(i), this.opt(i)); + } + return jo; + } + + + /** + * Make a JSON text of this JSONArray. For compactness, no + * unnecessary whitespace is added. If it is not possible to produce a + * syntactically correct JSON text then null will be returned instead. This + * could occur if the array contains an invalid number. + *

+ * Warning: This method assumes that the data structure is acyclical. + * + * @return a printable, displayable, transmittable + * representation of the array. + */ + public String toString() { + try { + return '[' + join(",") + ']'; + } catch (Exception e) { + return null; + } + } + + + /** + * Make a prettyprinted JSON text of this JSONArray. + * Warning: This method assumes that the data structure is acyclical. + * @param indentFactor The number of spaces to add to each level of + * indentation. + * @return a printable, displayable, transmittable + * representation of the object, beginning + * with [ (left bracket) and ending + * with ] (right bracket). + * @throws JSONException + */ + public String toString(int indentFactor) throws JSONException { + return toString(indentFactor, 0); + } + + + /** + * Make a prettyprinted JSON text of this JSONArray. + * Warning: This method assumes that the data structure is acyclical. + * @param indentFactor The number of spaces to add to each level of + * indentation. + * @param indent The indention of the top level. + * @return a printable, displayable, transmittable + * representation of the array. + * @throws JSONException + */ + String toString(int indentFactor, int indent) throws JSONException { + int len = length(); + if (len == 0) { + return "[]"; + } + int i; + StringBuffer sb = new StringBuffer("["); + if (len == 1) { + sb.append(JSONObject.valueToString(this.myArrayList.get(0), + indentFactor, indent)); + } else { + int newindent = indent + indentFactor; + sb.append('\n'); + for (i = 0; i < len; i += 1) { + if (i > 0) { + sb.append(",\n"); + } + for (int j = 0; j < newindent; j += 1) { + sb.append(' '); + } + sb.append(JSONObject.valueToString(this.myArrayList.get(i), + indentFactor, newindent)); + } + sb.append('\n'); + for (i = 0; i < indent; i += 1) { + sb.append(' '); + } + } + sb.append(']'); + return sb.toString(); + } + + + /** + * Write the contents of the JSONArray as JSON text to a writer. + * For compactness, no whitespace is added. + *

+ * Warning: This method assumes that the data structure is acyclical. + * + * @return The writer. + * @throws JSONException + */ + public Writer write(Writer writer) throws JSONException { + try { + boolean b = false; + int len = length(); + + writer.write('['); + + for (int i = 0; i < len; i += 1) { + if (b) { + writer.write(','); + } + Object v = this.myArrayList.get(i); + if (v instanceof JSONObject) { + ((JSONObject)v).write(writer); + } else if (v instanceof JSONArray) { + ((JSONArray)v).write(writer); + } else { + writer.write(JSONObject.valueToString(v)); + } + b = true; + } + writer.write(']'); + return writer; + } catch (IOException e) { + throw new JSONException(e); + } + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/json/JSONException.java b/commons/src/main/java/org/json/JSONException.java new file mode 100644 index 00000000..21255297 --- /dev/null +++ b/commons/src/main/java/org/json/JSONException.java @@ -0,0 +1,28 @@ +package org.json; + +/** + * The JSONException is thrown by the JSON.org classes then things are amiss. + * @author JSON.org + * @version 2 + */ +@SuppressWarnings({ "unchecked", "serial" }) +public class JSONException extends Exception { + private Throwable cause; + + /** + * Constructs a JSONException with an explanatory message. + * @param message Detail about the reason for the exception. + */ + public JSONException(String message) { + super(message); + } + + public JSONException(Throwable t) { + super(t.getMessage()); + this.cause = t; + } + + public Throwable getCause() { + return this.cause; + } +} diff --git a/commons/src/main/java/org/json/JSONObject.java b/commons/src/main/java/org/json/JSONObject.java new file mode 100644 index 00000000..96aac171 --- /dev/null +++ b/commons/src/main/java/org/json/JSONObject.java @@ -0,0 +1,1328 @@ +package org.json; + +/* +Copyright (c) 2002 JSON.org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +The Software shall be used for Good, not Evil. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +import java.io.IOException; +import java.io.Writer; +import java.util.Collection; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +/** + * A JSONObject is an unordered collection of name/value pairs. Its + * external form is a string wrapped in curly braces with colons between the + * names and values, and commas between the values and names. The internal form + * is an object having get and opt methods for + * accessing the values by name, and put methods for adding or + * replacing values by name. The values can be any of these types: + * Boolean, JSONArray, JSONObject, + * Number, String, or the JSONObject.NULL + * object. A JSONObject constructor can be used to convert an external form + * JSON text into an internal form whose values can be retrieved with the + * get and opt methods, or to convert values into a + * JSON text using the put and toString methods. + * A get method returns a value if one can be found, and throws an + * exception if one cannot be found. An opt method returns a + * default value instead of throwing an exception, and so is useful for + * obtaining optional values. + *

+ * The generic get() and opt() methods return an + * object, which you can cast or query for type. There are also typed + * get and opt methods that do type checking and type + * coersion for you. + *

+ * The put methods adds values to an object. For example, 

+ *     myString = new JSONObject().put("JSON", "Hello, World!").toString();
+ * produces the string {"JSON": "Hello, World"}. + *

+ * The texts produced by the toString methods strictly conform to + * the JSON sysntax rules. + * The constructors are more forgiving in the texts they will accept: + *

    + *
  • An extra , (comma) may appear just + * before the closing brace.
    • + *
  • Strings may be quoted with ' (single + * quote).
    • + *
  • Strings do not need to be quoted at all if they do not begin with a quote + * or single quote, and if they do not contain leading or trailing spaces, + * and if they do not contain any of these characters: + * { } [ ] / \ : , = ; # and if they do not look like numbers + * and if they are not the reserved words true, + * false, or null.
    • + *
  • Keys can be followed by = or => as well as + * by :.
    • + *
  • Values can be followed by ; (semicolon) as + * well as by , (comma).
    • + *
  • Numbers may have the 0- (octal) or + * 0x- (hex) prefix.
    • + *
  • Comments written in the slashshlash, slashstar, and hash conventions + * will be ignored.
    • + *
+ * @author JSON.org + * @version 2 + */ +@SuppressWarnings("unchecked") +public class JSONObject { + + /** + * JSONObject.NULL is equivalent to the value that JavaScript calls null, + * whilst Java's null is equivalent to the value that JavaScript calls + * undefined. + */ + private static final class Null { + + /** + * There is only intended to be a single instance of the NULL object, + * so the clone method returns itself. + * @return NULL. + */ + protected final Object clone() { + return this; + } + + + /** + * A Null object is equal to the null value and to itself. + * @param object An object to test for nullness. + * @return true if the object parameter is the JSONObject.NULL object + * or null. + */ + public boolean equals(Object object) { + return object == null || object == this; + } + + + /** + * Get the "null" string value. + * @return The string "null". + */ + public String toString() { + return "null"; + } + } + + + /** + * The hash map where the JSONObject's properties are kept. + */ + private HashMap myHashMap; + + + /** + * It is sometimes more convenient and less ambiguous to have a + * NULL object than to use Java's null value. + * JSONObject.NULL.equals(null) returns true. + * JSONObject.NULL.toString() returns "null". + */ + public static final Object NULL = new Null(); + + + /** + * Construct an empty JSONObject. + */ + public JSONObject() { + this.myHashMap = new HashMap(); + } + + + /** + * Construct a JSONObject from a subset of another JSONObject. + * An array of strings is used to identify the keys that should be copied. + * Missing keys are ignored. + * @param jo A JSONObject. + * @param sa An array of strings. + * @exception JSONException If a value is a non-finite number. + */ + public JSONObject(JSONObject jo, String[] sa) throws JSONException { + this(); + for (int i = 0; i < sa.length; i += 1) { + putOpt(sa[i], jo.opt(sa[i])); + } + } + + + /** + * Construct a JSONObject from a JSONTokener. + * @param x A JSONTokener object containing the source string. + * @throws JSONException If there is a syntax error in the source string. + */ + public JSONObject(JSONTokener x) throws JSONException { + this(); + char c; + String key; + + if (x.nextClean() != '{') { + throw x.syntaxError("A JSONObject text must begin with '{'"); + } + for (;;) { + c = x.nextClean(); + switch (c) { + case 0: + throw x.syntaxError("A JSONObject text must end with '}'"); + case '}': + return; + default: + x.back(); + key = x.nextValue().toString(); + } + + /* + * The key is followed by ':'. We will also tolerate '=' or '=>'. + */ + + c = x.nextClean(); + if (c == '=') { + if (x.next() != '>') { + x.back(); + } + } else if (c != ':') { + throw x.syntaxError("Expected a ':' after a key"); + } + put(key, x.nextValue()); + + /* + * Pairs are separated by ','. We will also tolerate ';'. + */ + + switch (x.nextClean()) { + case ';': + case ',': + if (x.nextClean() == '}') { + return; + } + x.back(); + break; + case '}': + return; + default: + throw x.syntaxError("Expected a ',' or '}'"); + } + } + } + + + /** + * Construct a JSONObject from a Map. + * @param map A map object that can be used to initialize the contents of + * the JSONObject. + */ + public JSONObject(Map map) { + this.myHashMap = (map == null) ? + new HashMap() : + new HashMap(map); + } + + + /** + * Construct a JSONObject from an Object using bean getters. + * It reflects on all of the public methods of the object. + * For each of the methods with no parameters and a name starting + * with "get" or "is" followed by an uppercase letter, + * the method is invoked, and the value + * name is put into the new JSONObject. + * + * The key is formed by removing the "get" or "is" prefix. If the second remaining + * character is not upper case, then the first + * character is converted to lower case. + * + * For example, if an object has a method named "getName", and + * if the result of calling object.getName() is "Larry Fine", + * then the JSONObject will contain "name": "Larry Fine". + * + * @param object An object that has getter methods that should be used + * to make a JSONObject. + */ + public JSONObject(Object object) { + this(); + Class c = object.getClass(); + Method[] methods = c.getMethods(); + for (int i = 0; i < methods.length; i += 1) { + try { + Method method = methods[i]; + String name = method.getName(); + String key = ""; + if (name.startsWith("get")) { + key = name.substring(3); + } else if (name.startsWith("is")) { + key = name.substring(2); + } + if (key.length() > 0 && + Character.isUpperCase(key.charAt(0)) && + method.getParameterTypes().length == 0) { + if (key.length() == 1) { + key = key.toLowerCase(); + } else if (!Character.isUpperCase(key.charAt(1))) { + key = key.substring(0, 1).toLowerCase() + + key.substring(1); + } + this.put(key, method.invoke(object, (Object[]) null)); + } + } catch (Exception e) { + /* forget about it */ + } + } + } + + /** + * Construct a JSONObject from an Object, using reflection to find the + * public members. The resulting JSONObject's keys will be the strings + * from the names array, and the values will be the field values associated + * with those keys in the object. If a key is not found or not visible, + * then it will not be copied into the new JSONObject. + * @param object An object that has fields that should be used to make a + * JSONObject. + * @param names An array of strings, the names of the fields to be obtained + * from the object. + */ + public JSONObject(Object object, String names[]) { + this(); + Class c = object.getClass(); + for (int i = 0; i < names.length; i += 1) { + try { + String name = names[i]; + Field field = c.getField(name); + Object value = field.get(object); + this.put(name, value); + } catch (Exception e) { + /* forget about it */ + } + } + } + + + /** + * Construct a JSONObject from a string. + * This is the most commonly used JSONObject constructor. + * @param string A string beginning + * with { (left brace) and ending + * with } (right brace). + * @exception JSONException If there is a syntax error in the source string. + */ + public JSONObject(String string) throws JSONException { + this(new JSONTokener(string)); + } + + + /** + * Accumulate values under a key. It is similar to the put method except + * that if there is already an object stored under the key then a + * JSONArray is stored under the key to hold all of the accumulated values. + * If there is already a JSONArray, then the new value is appended to it. + * In contrast, the put method replaces the previous value. + * @param key A key string. + * @param value An object to be accumulated under the key. + * @return this. + * @throws JSONException If the value is an invalid number + * or if the key is null. + */ + public JSONObject accumulate(String key, Object value) + throws JSONException { + testValidity(value); + Object o = opt(key); + if (o == null) { + put(key, value instanceof JSONArray ? + new JSONArray().put(value) : + value); + } else if (o instanceof JSONArray) { + ((JSONArray)o).put(value); + } else { + put(key, new JSONArray().put(o).put(value)); + } + return this; + } + + + /** + * Append values to the array under a key. If the key does not exist in the + * JSONObject, then the key is put in the JSONObject with its value being a + * JSONArray containing the value parameter. If the key was already + * associated with a JSONArray, then the value parameter is appended to it. + * @param key A key string. + * @param value An object to be accumulated under the key. + * @return this. + * @throws JSONException If the key is null or if the current value + * associated with the key is not a JSONArray. + */ + public JSONObject append(String key, Object value) + throws JSONException { + testValidity(value); + Object o = opt(key); + if (o == null) { + put(key, new JSONArray().put(value)); + } else if (o instanceof JSONArray) { + put(key, ((JSONArray)o).put(value)); + } else { + throw new JSONException("JSONObject[" + key + + "] is not a JSONArray."); + } + return this; + } + + + /** + * Produce a string from a double. The string "null" will be returned if + * the number is not finite. + * @param d A double. + * @return A String. + */ + static public String doubleToString(double d) { + if (Double.isInfinite(d) || Double.isNaN(d)) { + return "null"; + } + +// Shave off trailing zeros and decimal point, if possible. + + String s = Double.toString(d); + if (s.indexOf('.') > 0 && s.indexOf('e') < 0 && s.indexOf('E') < 0) { + while (s.endsWith("0")) { + s = s.substring(0, s.length() - 1); + } + if (s.endsWith(".")) { + s = s.substring(0, s.length() - 1); + } + } + return s; + } + + + /** + * Get the value object associated with a key. + * + * @param key A key string. + * @return The object associated with the key. + * @throws JSONException if the key is not found. + */ + public Object get(String key) throws JSONException { + Object o = opt(key); + if (o == null) { + throw new JSONException("JSONObject[" + quote(key) + + "] not found."); + } + return o; + } + + + /** + * Get the boolean value associated with a key. + * + * @param key A key string. + * @return The truth. + * @throws JSONException + * if the value is not a Boolean or the String "true" or "false". + */ + public boolean getBoolean(String key) throws JSONException { + Object o = get(key); + if (o.equals(Boolean.FALSE) || + (o instanceof String && + ((String)o).equalsIgnoreCase("false"))) { + return false; + } else if (o.equals(Boolean.TRUE) || + (o instanceof String && + ((String)o).equalsIgnoreCase("true"))) { + return true; + } + throw new JSONException("JSONObject[" + quote(key) + + "] is not a Boolean."); + } + + + /** + * Get the double value associated with a key. + * @param key A key string. + * @return The numeric value. + * @throws JSONException if the key is not found or + * if the value is not a Number object and cannot be converted to a number. + */ + public double getDouble(String key) throws JSONException { + Object o = get(key); + try { + return o instanceof Number ? + ((Number)o).doubleValue() : + Double.valueOf((String)o).doubleValue(); + } catch (Exception e) { + throw new JSONException("JSONObject[" + quote(key) + + "] is not a number."); + } + } + + + /** + * Get the int value associated with a key. If the number value is too + * large for an int, it will be clipped. + * + * @param key A key string. + * @return The integer value. + * @throws JSONException if the key is not found or if the value cannot + * be converted to an integer. + */ + public int getInt(String key) throws JSONException { + Object o = get(key); + return o instanceof Number ? + ((Number)o).intValue() : (int)getDouble(key); + } + + + /** + * Get the JSONArray value associated with a key. + * + * @param key A key string. + * @return A JSONArray which is the value. + * @throws JSONException if the key is not found or + * if the value is not a JSONArray. + */ + public JSONArray getJSONArray(String key) throws JSONException { + Object o = get(key); + if (o instanceof JSONArray) { + return (JSONArray)o; + } + throw new JSONException("JSONObject[" + quote(key) + + "] is not a JSONArray."); + } + + + /** + * Get the JSONObject value associated with a key. + * + * @param key A key string. + * @return A JSONObject which is the value. + * @throws JSONException if the key is not found or + * if the value is not a JSONObject. + */ + public JSONObject getJSONObject(String key) throws JSONException { + Object o = get(key); + if (o instanceof JSONObject) { + return (JSONObject)o; + } + throw new JSONException("JSONObject[" + quote(key) + + "] is not a JSONObject."); + } + + + /** + * Get the long value associated with a key. If the number value is too + * long for a long, it will be clipped. + * + * @param key A key string. + * @return The long value. + * @throws JSONException if the key is not found or if the value cannot + * be converted to a long. + */ + public long getLong(String key) throws JSONException { + Object o = get(key); + return o instanceof Number ? + ((Number)o).longValue() : (long)getDouble(key); + } + + + /** + * Get the string associated with a key. + * + * @param key A key string. + * @return A string which is the value. + * @throws JSONException if the key is not found. + */ + public String getString(String key) throws JSONException { + return get(key).toString(); + } + + + /** + * Determine if the JSONObject contains a specific key. + * @param key A key string. + * @return true if the key exists in the JSONObject. + */ + public boolean has(String key) { + return this.myHashMap.containsKey(key); + } + + + /** + * Determine if the value associated with the key is null or if there is + * no value. + * @param key A key string. + * @return true if there is no value associated with the key or if + * the value is the JSONObject.NULL object. + */ + public boolean isNull(String key) { + return JSONObject.NULL.equals(opt(key)); + } + + + /** + * Get an enumeration of the keys of the JSONObject. + * + * @return An iterator of the keys. + */ + public Iterator keys() { + return this.myHashMap.keySet().iterator(); + } + + + /** + * Get the number of keys stored in the JSONObject. + * + * @return The number of keys in the JSONObject. + */ + public int length() { + return this.myHashMap.size(); + } + + + /** + * Produce a JSONArray containing the names of the elements of this + * JSONObject. + * @return A JSONArray containing the key strings, or null if the JSONObject + * is empty. + */ + public JSONArray names() { + JSONArray ja = new JSONArray(); + Iterator keys = keys(); + while (keys.hasNext()) { + ja.put(keys.next()); + } + return ja.length() == 0 ? null : ja; + } + + /** + * Produce a string from a Number. + * @param n A Number + * @return A String. + * @throws JSONException If n is a non-finite number. + */ + static public String numberToString(Number n) + throws JSONException { + if (n == null) { + throw new JSONException("Null pointer"); + } + testValidity(n); + +// Shave off trailing zeros and decimal point, if possible. + + String s = n.toString(); + if (s.indexOf('.') > 0 && s.indexOf('e') < 0 && s.indexOf('E') < 0) { + while (s.endsWith("0")) { + s = s.substring(0, s.length() - 1); + } + if (s.endsWith(".")) { + s = s.substring(0, s.length() - 1); + } + } + return s; + } + + + /** + * Get an optional value associated with a key. + * @param key A key string. + * @return An object which is the value, or null if there is no value. + */ + public Object opt(String key) { + return key == null ? null : this.myHashMap.get(key); + } + + + /** + * Get an optional boolean associated with a key. + * It returns false if there is no such key, or if the value is not + * Boolean.TRUE or the String "true". + * + * @param key A key string. + * @return The truth. + */ + public boolean optBoolean(String key) { + return optBoolean(key, false); + } + + + /** + * Get an optional boolean associated with a key. + * It returns the defaultValue if there is no such key, or if it is not + * a Boolean or the String "true" or "false" (case insensitive). + * + * @param key A key string. + * @param defaultValue The default. + * @return The truth. + */ + public boolean optBoolean(String key, boolean defaultValue) { + try { + return getBoolean(key); + } catch (Exception e) { + return defaultValue; + } + } + + + /** + * Put a key/value pair in the JSONObject, where the value will be a + * JSONArray which is produced from a Collection. + * @param key A key string. + * @param value A Collection value. + * @return this. + * @throws JSONException + */ + public JSONObject put(String key, Collection value) throws JSONException { + put(key, new JSONArray(value)); + return this; + } + + + /** + * Get an optional double associated with a key, + * or NaN if there is no such key or if its value is not a number. + * If the value is a string, an attempt will be made to evaluate it as + * a number. + * + * @param key A string which is the key. + * @return An object which is the value. + */ + public double optDouble(String key) { + return optDouble(key, Double.NaN); + } + + + /** + * Get an optional double associated with a key, or the + * defaultValue if there is no such key or if its value is not a number. + * If the value is a string, an attempt will be made to evaluate it as + * a number. + * + * @param key A key string. + * @param defaultValue The default. + * @return An object which is the value. + */ + public double optDouble(String key, double defaultValue) { + try { + Object o = opt(key); + return o instanceof Number ? ((Number)o).doubleValue() : + new Double((String)o).doubleValue(); + } catch (Exception e) { + return defaultValue; + } + } + + + /** + * Get an optional int value associated with a key, + * or zero if there is no such key or if the value is not a number. + * If the value is a string, an attempt will be made to evaluate it as + * a number. + * + * @param key A key string. + * @return An object which is the value. + */ + public int optInt(String key) { + return optInt(key, 0); + } + + + /** + * Get an optional int value associated with a key, + * or the default if there is no such key or if the value is not a number. + * If the value is a string, an attempt will be made to evaluate it as + * a number. + * + * @param key A key string. + * @param defaultValue The default. + * @return An object which is the value. + */ + public int optInt(String key, int defaultValue) { + try { + return getInt(key); + } catch (Exception e) { + return defaultValue; + } + } + + + /** + * Get an optional JSONArray associated with a key. + * It returns null if there is no such key, or if its value is not a + * JSONArray. + * + * @param key A key string. + * @return A JSONArray which is the value. + */ + public JSONArray optJSONArray(String key) { + Object o = opt(key); + return o instanceof JSONArray ? (JSONArray)o : null; + } + + + /** + * Get an optional JSONObject associated with a key. + * It returns null if there is no such key, or if its value is not a + * JSONObject. + * + * @param key A key string. + * @return A JSONObject which is the value. + */ + public JSONObject optJSONObject(String key) { + Object o = opt(key); + return o instanceof JSONObject ? (JSONObject)o : null; + } + + + /** + * Get an optional long value associated with a key, + * or zero if there is no such key or if the value is not a number. + * If the value is a string, an attempt will be made to evaluate it as + * a number. + * + * @param key A key string. + * @return An object which is the value. + */ + public long optLong(String key) { + return optLong(key, 0); + } + + + /** + * Get an optional long value associated with a key, + * or the default if there is no such key or if the value is not a number. + * If the value is a string, an attempt will be made to evaluate it as + * a number. + * + * @param key A key string. + * @param defaultValue The default. + * @return An object which is the value. + */ + public long optLong(String key, long defaultValue) { + try { + return getLong(key); + } catch (Exception e) { + return defaultValue; + } + } + + + /** + * Get an optional string associated with a key. + * It returns an empty string if there is no such key. If the value is not + * a string and is not null, then it is coverted to a string. + * + * @param key A key string. + * @return A string which is the value. + */ + public String optString(String key) { + return optString(key, ""); + } + + + /** + * Get an optional string associated with a key. + * It returns the defaultValue if there is no such key. + * + * @param key A key string. + * @param defaultValue The default. + * @return A string which is the value. + */ + public String optString(String key, String defaultValue) { + Object o = opt(key); + return o != null ? o.toString() : defaultValue; + } + + + /** + * Put a key/boolean pair in the JSONObject. + * + * @param key A key string. + * @param value A boolean which is the value. + * @return this. + * @throws JSONException If the key is null. + */ + public JSONObject put(String key, boolean value) throws JSONException { + put(key, value ? Boolean.TRUE : Boolean.FALSE); + return this; + } + + + /** + * Put a key/double pair in the JSONObject. + * + * @param key A key string. + * @param value A double which is the value. + * @return this. + * @throws JSONException If the key is null or if the number is invalid. + */ + public JSONObject put(String key, double value) throws JSONException { + put(key, new Double(value)); + return this; + } + + + /** + * Put a key/int pair in the JSONObject. + * + * @param key A key string. + * @param value An int which is the value. + * @return this. + * @throws JSONException If the key is null. + */ + public JSONObject put(String key, int value) throws JSONException { + put(key, new Integer(value)); + return this; + } + + + /** + * Put a key/long pair in the JSONObject. + * + * @param key A key string. + * @param value A long which is the value. + * @return this. + * @throws JSONException If the key is null. + */ + public JSONObject put(String key, long value) throws JSONException { + put(key, new Long(value)); + return this; + } + + + /** + * Put a key/value pair in the JSONObject, where the value will be a + * JSONObject which is produced from a Map. + * @param key A key string. + * @param value A Map value. + * @return this. + * @throws JSONException + */ + public JSONObject put(String key, Map value) throws JSONException { + put(key, new JSONObject(value)); + return this; + } + + + /** + * Put a key/value pair in the JSONObject. If the value is null, + * then the key will be removed from the JSONObject if it is present. + * @param key A key string. + * @param value An object which is the value. It should be of one of these + * types: Boolean, Double, Integer, JSONArray, JSONObject, Long, String, + * or the JSONObject.NULL object. + * @return this. + * @throws JSONException If the value is non-finite number + * or if the key is null. + */ + public JSONObject put(String key, Object value) throws JSONException { + if (key == null) { + throw new JSONException("Null key."); + } + if (value != null) { + testValidity(value); + this.myHashMap.put(key, value); + } else { + remove(key); + } + return this; + } + + + /** + * Put a key/value pair in the JSONObject, but only if the + * key and the value are both non-null. + * @param key A key string. + * @param value An object which is the value. It should be of one of these + * types: Boolean, Double, Integer, JSONArray, JSONObject, Long, String, + * or the JSONObject.NULL object. + * @return this. + * @throws JSONException If the value is a non-finite number. + */ + public JSONObject putOpt(String key, Object value) throws JSONException { + if (key != null && value != null) { + put(key, value); + } + return this; + } + + + /** + * Produce a string in double quotes with backslash sequences in all the + * right places. A backslash will be inserted within = '\u0080' && c < '\u00a0') || + (c >= '\u2000' && c < '\u2100')) { + t = "000" + Integer.toHexString(c); + sb.append("\\u" + t.substring(t.length() - 4)); + } else { + sb.append(c); + } + } + } + sb.append('"'); + return sb.toString(); + } + + /** + * Remove a name and its value, if present. + * @param key The name to be removed. + * @return The value that was associated with the name, + * or null if there was no value. + */ + public Object remove(String key) { + return this.myHashMap.remove(key); + } + + + /** + * Throw an exception if the object is an NaN or infinite number. + * @param o The object to test. + * @throws JSONException If o is a non-finite number. + */ + static void testValidity(Object o) throws JSONException { + if (o != null) { + if (o instanceof Double) { + if (((Double)o).isInfinite() || ((Double)o).isNaN()) { + throw new JSONException( + "JSON does not allow non-finite numbers."); + } + } else if (o instanceof Float) { + if (((Float)o).isInfinite() || ((Float)o).isNaN()) { + throw new JSONException( + "JSON does not allow non-finite numbers."); + } + } + } + } + + + /** + * Produce a JSONArray containing the values of the members of this + * JSONObject. + * @param names A JSONArray containing a list of key strings. This + * determines the sequence of the values in the result. + * @return A JSONArray of values. + * @throws JSONException If any of the values are non-finite numbers. + */ + public JSONArray toJSONArray(JSONArray names) throws JSONException { + if (names == null || names.length() == 0) { + return null; + } + JSONArray ja = new JSONArray(); + for (int i = 0; i < names.length(); i += 1) { + ja.put(this.opt(names.getString(i))); + } + return ja; + } + + /** + * Make a JSON text of this JSONObject. For compactness, no whitespace + * is added. If this would not result in a syntactically correct JSON text, + * then null will be returned instead. + *

+ * Warning: This method assumes that the data structure is acyclical. + * + * @return a printable, displayable, portable, transmittable + * representation of the object, beginning + * with { (left brace) and ending + * with } (right brace). + */ + public String toString() { + try { + Iterator keys = keys(); + StringBuffer sb = new StringBuffer("{"); + + while (keys.hasNext()) { + if (sb.length() > 1) { + sb.append(','); + } + Object o = keys.next(); + sb.append(quote(o.toString())); + sb.append(':'); + sb.append(valueToString(this.myHashMap.get(o))); + } + sb.append('}'); + return sb.toString(); + } catch (Exception e) { + return null; + } + } + + + /** + * Make a prettyprinted JSON text of this JSONObject. + *

+ * Warning: This method assumes that the data structure is acyclical. + * @param indentFactor The number of spaces to add to each level of + * indentation. + * @return a printable, displayable, portable, transmittable + * representation of the object, beginning + * with { (left brace) and ending + * with } (right brace). + * @throws JSONException If the object contains an invalid number. + */ + public String toString(int indentFactor) throws JSONException { + return toString(indentFactor, 0); + } + + + /** + * Make a prettyprinted JSON text of this JSONObject. + *

+ * Warning: This method assumes that the data structure is acyclical. + * @param indentFactor The number of spaces to add to each level of + * indentation. + * @param indent The indentation of the top level. + * @return a printable, displayable, transmittable + * representation of the object, beginning + * with { (left brace) and ending + * with } (right brace). + * @throws JSONException If the object contains an invalid number. + */ + String toString(int indentFactor, int indent) throws JSONException { + int i; + int n = length(); + if (n == 0) { + return "{}"; + } + Iterator keys = keys(); + StringBuffer sb = new StringBuffer("{"); + int newindent = indent + indentFactor; + Object o; + if (n == 1) { + o = keys.next(); + sb.append(quote(o.toString())); + sb.append(": "); + sb.append(valueToString(this.myHashMap.get(o), indentFactor, + indent)); + } else { + while (keys.hasNext()) { + o = keys.next(); + if (sb.length() > 1) { + sb.append(",\n"); + } else { + sb.append('\n'); + } + for (i = 0; i < newindent; i += 1) { + sb.append(' '); + } + sb.append(quote(o.toString())); + sb.append(": "); + sb.append(valueToString(this.myHashMap.get(o), indentFactor, + newindent)); + } + if (sb.length() > 1) { + sb.append('\n'); + for (i = 0; i < indent; i += 1) { + sb.append(' '); + } + } + } + sb.append('}'); + return sb.toString(); + } + + + /** + * Make a JSON text of an Object value. If the object has an + * value.toJSONString() method, then that method will be used to produce + * the JSON text. The method is required to produce a strictly + * conforming text. If the object does not contain a toJSONString + * method (which is the most common case), then a text will be + * produced by other means. If the value is an array or Collection, + * then a JSONArray will be made from it and its toJSONString method + * will be called. If the value is a MAP, then a JSONObject will be made + * from it and its toJSONString method will be called. Otherwise, the + * value's toString method will be called, and the result will be quoted. + * + *

+ * Warning: This method assumes that the data structure is acyclical. + * @param value The value to be serialized. + * @return a printable, displayable, transmittable + * representation of the object, beginning + * with { (left brace) and ending + * with } (right brace). + * @throws JSONException If the value is or contains an invalid number. + */ + static String valueToString(Object value) throws JSONException { + if (value == null || value.equals(null)) { + return "null"; + } + if (value instanceof JSONString) { + Object o; + try { + o = ((JSONString)value).toJSONString(); + } catch (Exception e) { + throw new JSONException(e); + } + if (o instanceof String) { + return (String)o; + } + throw new JSONException("Bad value from toJSONString: " + o); + } + if (value instanceof Number) { + return numberToString((Number) value); + } + if (value instanceof Boolean || value instanceof JSONObject || + value instanceof JSONArray) { + return value.toString(); + } + if (value instanceof Map) { + return new JSONObject((Map)value).toString(); + } + if (value instanceof Collection) { + return new JSONArray((Collection)value).toString(); + } + if (value.getClass().isArray()) { + return new JSONArray(value).toString(); + } + return quote(value.toString()); + } + + + /** + * Make a prettyprinted JSON text of an object value. + *

+ * Warning: This method assumes that the data structure is acyclical. + * @param value The value to be serialized. + * @param indentFactor The number of spaces to add to each level of + * indentation. + * @param indent The indentation of the top level. + * @return a printable, displayable, transmittable + * representation of the object, beginning + * with { (left brace) and ending + * with } (right brace). + * @throws JSONException If the object contains an invalid number. + */ + static String valueToString(Object value, int indentFactor, int indent) + throws JSONException { + if (value == null || value.equals(null)) { + return "null"; + } + try { + if (value instanceof JSONString) { + Object o = ((JSONString)value).toJSONString(); + if (o instanceof String) { + return (String)o; + } + } + } catch (Exception e) { + /* forget about it */ + } + if (value instanceof Number) { + return numberToString((Number) value); + } + if (value instanceof Boolean) { + return value.toString(); + } + if (value instanceof JSONObject) { + return ((JSONObject)value).toString(indentFactor, indent); + } + if (value instanceof JSONArray) { + return ((JSONArray)value).toString(indentFactor, indent); + } + return quote(value.toString()); + } + + + /** + * Write the contents of the JSONObject as JSON text to a writer. + * For compactness, no whitespace is added. + *

+ * Warning: This method assumes that the data structure is acyclical. + * + * @return The writer. + * @throws JSONException + */ + public Writer write(Writer writer) throws JSONException { + try { + boolean b = false; + Iterator keys = keys(); + writer.write('{'); + + while (keys.hasNext()) { + if (b) { + writer.write(','); + } + Object k = keys.next(); + writer.write(quote(k.toString())); + writer.write(':'); + Object v = this.myHashMap.get(k); + if (v instanceof JSONObject) { + ((JSONObject)v).write(writer); + } else if (v instanceof JSONArray) { + ((JSONArray)v).write(writer); + } else { + writer.write(valueToString(v)); + } + b = true; + } + writer.write('}'); + return writer; + } catch (IOException e) { + throw new JSONException(e); + } + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/json/JSONString.java b/commons/src/main/java/org/json/JSONString.java new file mode 100644 index 00000000..7f4f65b6 --- /dev/null +++ b/commons/src/main/java/org/json/JSONString.java @@ -0,0 +1,18 @@ +package org.json; +/** + * The JSONString interface allows a toJSONString() + * method so that a class can change the behavior of + * JSONObject.toString(), JSONArray.toString(), + * and JSONWriter.value(Object). The + * toJSONString method will be used instead of the default behavior + * of using the Object's toString() method and quoting the result. + */ +public interface JSONString { + /** + * The toJSONString method allows a class to produce its own JSON + * serialization. + * + * @return A strictly syntactically correct JSON text. + */ + public String toJSONString(); +} diff --git a/commons/src/main/java/org/json/JSONStringer.java b/commons/src/main/java/org/json/JSONStringer.java new file mode 100644 index 00000000..b68efa41 --- /dev/null +++ b/commons/src/main/java/org/json/JSONStringer.java @@ -0,0 +1,78 @@ +package org.json; + +/* +Copyright (c) 2006 JSON.org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +The Software shall be used for Good, not Evil. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +import java.io.StringWriter; + +/** + * JSONStringer provides a quick and convenient way of producing JSON text. + * The texts produced strictly conform to JSON syntax rules. No whitespace is + * added, so the results are ready for transmission or storage. Each instance of + * JSONStringer can produce one JSON text. + *

+ * A JSONStringer instance provides a value method for appending + * values to the + * text, and a key + * method for adding keys before values in objects. There are array + * and endArray methods that make and bound array values, and + * object and endObject methods which make and bound + * object values. All of these methods return the JSONWriter instance, + * permitting cascade style. For example, 

+ * myString = new JSONStringer()
+ *     .object()
+ *         .key("JSON")
+ *         .value("Hello, World!")
+ *     .endObject()
+ *     .toString();
which produces the string 
+ * {"JSON":"Hello, World!"}
+ *

+ * The first method called must be array or object. + * There are no methods for adding commas or colons. JSONStringer adds them for + * you. Objects and arrays can be nested up to 20 levels deep. + *

+ * This can sometimes be easier than using a JSONObject to build a string. + * @author JSON.org + * @version 2 + */ +public class JSONStringer extends JSONWriter { + /** + * Make a fresh JSONStringer. It can be used to build one JSON text. + */ + public JSONStringer() { + super(new StringWriter()); + } + + /** + * Return the JSON text. This method is used to obtain the product of the + * JSONStringer instance. It will return null if there was a + * problem in the construction of the JSON text (such as the calls to + * array were not properly balanced with calls to + * endArray). + * @return The JSON text. + */ + public String toString() { + return this.mode == 'd' ? this.writer.toString() : null; + } +} diff --git a/commons/src/main/java/org/json/JSONTokener.java b/commons/src/main/java/org/json/JSONTokener.java new file mode 100644 index 00000000..3bf5ca9d --- /dev/null +++ b/commons/src/main/java/org/json/JSONTokener.java @@ -0,0 +1,462 @@ +package org.json; + +/* +Copyright (c) 2002 JSON.org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +The Software shall be used for Good, not Evil. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +/** + * A JSONTokener takes a source string and extracts characters and tokens from + * it. It is used by the JSONObject and JSONArray constructors to parse + * JSON source strings. + * @author JSON.org + * @version 2 + */ +public class JSONTokener { + + /** + * The index of the next character. + */ + private int myIndex; + + + /** + * The source string being tokenized. + */ + private String mySource; + + + /** + * Construct a JSONTokener from a string. + * + * @param s A source string. + */ + public JSONTokener(String s) { + this.myIndex = 0; + this.mySource = s; + } + + + /** + * Back up one character. This provides a sort of lookahead capability, + * so that you can test for a digit or letter before attempting to parse + * the next number or identifier. + */ + public void back() { + if (this.myIndex > 0) { + this.myIndex -= 1; + } + } + + + + /** + * Get the hex value of a character (base16). + * @param c A character between '0' and '9' or between 'A' and 'F' or + * between 'a' and 'f'. + * @return An int between 0 and 15, or -1 if c was not a hex digit. + */ + public static int dehexchar(char c) { + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'A' && c <= 'F') { + return c - ('A' - 10); + } + if (c >= 'a' && c <= 'f') { + return c - ('a' - 10); + } + return -1; + } + + + /** + * Determine if the source string still contains characters that next() + * can consume. + * @return true if not yet at the end of the source. + */ + public boolean more() { + return this.myIndex < this.mySource.length(); + } + + + /** + * Get the next character in the source string. + * + * @return The next character, or 0 if past the end of the source string. + */ + public char next() { + if (more()) { + char c = this.mySource.charAt(this.myIndex); + this.myIndex += 1; + return c; + } + return 0; + } + + + /** + * Consume the next character, and check that it matches a specified + * character. + * @param c The character to match. + * @return The character. + * @throws JSONException if the character does not match. + */ + public char next(char c) throws JSONException { + char n = next(); + if (n != c) { + throw syntaxError("Expected '" + c + "' and instead saw '" + + n + "'"); + } + return n; + } + + + /** + * Get the next n characters. + * + * @param n The number of characters to take. + * @return A string of n characters. + * @throws JSONException + * Substring bounds error if there are not + * n characters remaining in the source string. + */ + public String next(int n) throws JSONException { + int i = this.myIndex; + int j = i + n; + if (j >= this.mySource.length()) { + throw syntaxError("Substring bounds error"); + } + this.myIndex += n; + return this.mySource.substring(i, j); + } + + + /** + * Get the next char in the string, skipping whitespace + * and comments (slashslash, slashstar, and hash). + * @throws JSONException + * @return A character, or 0 if there are no more characters. + */ + public char nextClean() throws JSONException { + for (;;) { + char c = next(); + if (c == '/') { + switch (next()) { + case '/': + do { + c = next(); + } while (c != '\n' && c != '\r' && c != 0); + break; + case '*': + for (;;) { + c = next(); + if (c == 0) { + throw syntaxError("Unclosed comment"); + } + if (c == '*') { + if (next() == '/') { + break; + } + back(); + } + } + break; + default: + back(); + return '/'; + } + } else if (c == '#') { + do { + c = next(); + } while (c != '\n' && c != '\r' && c != 0); + } else if (c == 0 || c > ' ') { + return c; + } + } + } + + + /** + * Return the characters up to the next close quote character. + * Backslash processing is done. The formal JSON format does not + * allow strings in single quotes, but an implementation is allowed to + * accept them. + * @param quote The quoting character, either + * " (double quote) or + * ' (single quote). + * @return A String. + * @throws JSONException Unterminated string. + */ + public String nextString(char quote) throws JSONException { + char c; + StringBuffer sb = new StringBuffer(); + for (;;) { + c = next(); + switch (c) { + case 0: + case '\n': + case '\r': + throw syntaxError("Unterminated string"); + case '\\': + c = next(); + switch (c) { + case 'b': + sb.append('\b'); + break; + case 't': + sb.append('\t'); + break; + case 'n': + sb.append('\n'); + break; + case 'f': + sb.append('\f'); + break; + case 'r': + sb.append('\r'); + break; + case 'u': + sb.append((char)Integer.parseInt(next(4), 16)); + break; + case 'x' : + sb.append((char) Integer.parseInt(next(2), 16)); + break; + default: + sb.append(c); + } + break; + default: + if (c == quote) { + return sb.toString(); + } + sb.append(c); + } + } + } + + + /** + * Get the text up but not including the specified character or the + * end of line, whichever comes first. + * @param d A delimiter character. + * @return A string. + */ + public String nextTo(char d) { + StringBuffer sb = new StringBuffer(); + for (;;) { + char c = next(); + if (c == d || c == 0 || c == '\n' || c == '\r') { + if (c != 0) { + back(); + } + return sb.toString().trim(); + } + sb.append(c); + } + } + + + /** + * Get the text up but not including one of the specified delimeter + * characters or the end of line, whichever comes first. + * @param delimiters A set of delimiter characters. + * @return A string, trimmed. + */ + public String nextTo(String delimiters) { + char c; + StringBuffer sb = new StringBuffer(); + for (;;) { + c = next(); + if (delimiters.indexOf(c) >= 0 || c == 0 || + c == '\n' || c == '\r') { + if (c != 0) { + back(); + } + return sb.toString().trim(); + } + sb.append(c); + } + } + + + /** + * Get the next value. The value can be a Boolean, Double, Integer, + * JSONArray, JSONObject, Long, or String, or the JSONObject.NULL object. + * @throws JSONException If syntax error. + * + * @return An object. + */ + public Object nextValue() throws JSONException { + char c = nextClean(); + String s; + + switch (c) { + case '"': + case '\'': + return nextString(c); + case '{': + back(); + return new JSONObject(this); + case '[': + back(); + return new JSONArray(this); + } + + /* + * Handle unquoted text. This could be the values true, false, or + * null, or it can be a number. An implementation (such as this one) + * is allowed to also accept non-standard forms. + * + * Accumulate characters until we reach the end of the text or a + * formatting character. + */ + + StringBuffer sb = new StringBuffer(); + char b = c; + while (c >= ' ' && ",:]}/\\\"[{;=#".indexOf(c) < 0) { + sb.append(c); + c = next(); + } + back(); + + /* + * If it is true, false, or null, return the proper value. + */ + + s = sb.toString().trim(); + if (s.equals("")) { + throw syntaxError("Missing value"); + } + if (s.equalsIgnoreCase("true")) { + return Boolean.TRUE; + } + if (s.equalsIgnoreCase("false")) { + return Boolean.FALSE; + } + if (s.equalsIgnoreCase("null")) { + return JSONObject.NULL; + } + + /* + * If it might be a number, try converting it. We support the 0- and 0x- + * conventions. If a number cannot be produced, then the value will just + * be a string. Note that the 0-, 0x-, plus, and implied string + * conventions are non-standard. A JSON parser is free to accept + * non-JSON forms as long as it accepts all correct JSON forms. + */ + + if ((b >= '0' && b <= '9') || b == '.' || b == '-' || b == '+') { + if (b == '0') { + if (s.length() > 2 && + (s.charAt(1) == 'x' || s.charAt(1) == 'X')) { + try { + return new Integer(Integer.parseInt(s.substring(2), + 16)); + } catch (Exception e) { + /* Ignore the error */ + } + } else { + try { + return new Integer(Integer.parseInt(s, 8)); + } catch (Exception e) { + /* Ignore the error */ + } + } + } + try { + return new Integer(s); + } catch (Exception e) { + try { + return new Long(s); + } catch (Exception f) { + try { + return new Double(s); + } catch (Exception g) { + return s; + } + } + } + } + return s; + } + + + /** + * Skip characters until the next character is the requested character. + * If the requested character is not found, no characters are skipped. + * @param to A character to skip to. + * @return The requested character, or zero if the requested character + * is not found. + */ + public char skipTo(char to) { + char c; + int index = this.myIndex; + do { + c = next(); + if (c == 0) { + this.myIndex = index; + return c; + } + } while (c != to); + back(); + return c; + } + + + /** + * Skip characters until past the requested string. + * If it is not found, we are left at the end of the source. + * @param to A string to skip past. + */ + public boolean skipPast(String to) { + this.myIndex = this.mySource.indexOf(to, this.myIndex); + if (this.myIndex < 0) { + this.myIndex = this.mySource.length(); + return false; + } + this.myIndex += to.length(); + return true; + + } + + + /** + * Make a JSONException to signal a syntax error. + * + * @param message The error message. + * @return A JSONException object, suitable for throwing + */ + public JSONException syntaxError(String message) { + return new JSONException(message + toString()); + } + + + /** + * Make a printable string of this JSONTokener. + * + * @return " at character [this.myIndex] of [this.mySource]" + */ + public String toString() { + return " at character " + this.myIndex + " of " + this.mySource; + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/json/JSONWriter.java b/commons/src/main/java/org/json/JSONWriter.java new file mode 100644 index 00000000..e1cd2e9b --- /dev/null +++ b/commons/src/main/java/org/json/JSONWriter.java @@ -0,0 +1,318 @@ +package org.json; + +import java.io.IOException; +import java.io.Writer; + +/* +Copyright (c) 2006 JSON.org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +The Software shall be used for Good, not Evil. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +/** + * JSONWriter provides a quick and convenient way of producing JSON text. + * The texts produced strictly conform to JSON syntax rules. No whitespace is + * added, so the results are ready for transmission or storage. Each instance of + * JSONWriter can produce one JSON text. + *

+ * A JSONWriter instance provides a value method for appending + * values to the + * text, and a key + * method for adding keys before values in objects. There are array + * and endArray methods that make and bound array values, and + * object and endObject methods which make and bound + * object values. All of these methods return the JSONWriter instance, + * permitting a cascade style. For example, 

+ * new JSONWriter(myWriter)
+ *     .object()
+ *         .key("JSON")
+ *         .value("Hello, World!")
+ *     .endObject();
which writes 
+ * {"JSON":"Hello, World!"}
+ *

+ * The first method called must be array or object. + * There are no methods for adding commas or colons. JSONWriter adds them for + * you. Objects and arrays can be nested up to 20 levels deep. + *

+ * This can sometimes be easier than using a JSONObject to build a string. + * @author JSON.org + * @version 2 + */ +public class JSONWriter { + private static final int maxdepth = 20; + + /** + * The comma flag determines if a comma should be output before the next + * value. + */ + private boolean comma; + + /** + * The current mode. Values: + * 'a' (array), + * 'd' (done), + * 'i' (initial), + * 'k' (key), + * 'o' (object). + */ + protected char mode; + + /** + * The object/array stack. + */ + private char stack[]; + + /** + * The stack top index. A value of 0 indicates that the stack is empty. + */ + private int top; + + /** + * The writer that will receive the output. + */ + protected Writer writer; + + /** + * Make a fresh JSONWriter. It can be used to build one JSON text. + */ + public JSONWriter(Writer w) { + this.comma = false; + this.mode = 'i'; + this.stack = new char[maxdepth]; + this.top = 0; + this.writer = w; + } + + /** + * Append a value. + * @param s A string value. + * @return this + * @throws JSONException If the value is out of sequence. + */ + private JSONWriter append(String s) throws JSONException { + if (s == null) { + throw new JSONException("Null pointer"); + } + if (this.mode == 'o' || this.mode == 'a') { + try { + if (this.comma && this.mode == 'a') { + this.writer.write(','); + } + this.writer.write(s); + } catch (IOException e) { + throw new JSONException(e); + } + if (this.mode == 'o') { + this.mode = 'k'; + } + this.comma = true; + return this; + } + throw new JSONException("Value out of sequence."); + } + + /** + * Begin appending a new array. All values until the balancing + * endArray will be appended to this array. The + * endArray method must be called to mark the array's end. + * @return this + * @throws JSONException If the nesting is too deep, or if the object is + * started in the wrong place (for example as a key or after the end of the + * outermost array or object). + */ + public JSONWriter array() throws JSONException { + if (this.mode == 'i' || this.mode == 'o' || this.mode == 'a') { + this.push('a'); + this.append("["); + this.comma = false; + return this; + } + throw new JSONException("Misplaced array."); + } + + /** + * End something. + * @param m Mode + * @param c Closing character + * @return this + * @throws JSONException If unbalanced. + */ + private JSONWriter end(char m, char c) throws JSONException { + if (this.mode != m) { + throw new JSONException(m == 'o' ? "Misplaced endObject." : + "Misplaced endArray."); + } + this.pop(m); + try { + this.writer.write(c); + } catch (IOException e) { + throw new JSONException(e); + } + this.comma = true; + return this; + } + + /** + * End an array. This method most be called to balance calls to + * array. + * @return this + * @throws JSONException If incorrectly nested. + */ + public JSONWriter endArray() throws JSONException { + return this.end('a', ']'); + } + + /** + * End an object. This method most be called to balance calls to + * object. + * @return this + * @throws JSONException If incorrectly nested. + */ + public JSONWriter endObject() throws JSONException { + return this.end('k', '}'); + } + + /** + * Append a key. The key will be associated with the next value. In an + * object, every value must be preceded by a key. + * @param s A key string. + * @return this + * @throws JSONException If the key is out of place. For example, keys + * do not belong in arrays or if the key is null. + */ + public JSONWriter key(String s) throws JSONException { + if (s == null) { + throw new JSONException("Null key."); + } + if (this.mode == 'k') { + try { + if (this.comma) { + this.writer.write(','); + } + this.writer.write(JSONObject.quote(s)); + this.writer.write(':'); + this.comma = false; + this.mode = 'o'; + return this; + } catch (IOException e) { + throw new JSONException(e); + } + } + throw new JSONException("Misplaced key."); + } + + + /** + * Begin appending a new object. All keys and values until the balancing + * endObject will be appended to this object. The + * endObject method must be called to mark the object's end. + * @return this + * @throws JSONException If the nesting is too deep, or if the object is + * started in the wrong place (for example as a key or after the end of the + * outermost array or object). + */ + public JSONWriter object() throws JSONException { + if (this.mode == 'i') { + this.mode = 'o'; + } + if (this.mode == 'o' || this.mode == 'a') { + this.append("{"); + this.push('k'); + this.comma = false; + return this; + } + throw new JSONException("Misplaced object."); + + } + + + /** + * Pop an array or object scope. + * @param c The scope to close. + * @throws JSONException If nesting is wrong. + */ + private void pop(char c) throws JSONException { + if (this.top <= 0 || this.stack[this.top - 1] != c) { + throw new JSONException("Nesting error."); + } + this.top -= 1; + this.mode = this.top == 0 ? 'd' : this.stack[this.top - 1]; + } + + /** + * Push an array or object scope. + * @param c The scope to open. + * @throws JSONException If nesting is too deep. + */ + private void push(char c) throws JSONException { + if (this.top >= maxdepth) { + throw new JSONException("Nesting too deep."); + } + this.stack[this.top] = c; + this.mode = c; + this.top += 1; + } + + + /** + * Append either the value true or the value + * false. + * @param b A boolean. + * @return this + * @throws JSONException + */ + public JSONWriter value(boolean b) throws JSONException { + return this.append(b ? "true" : "false"); + } + + /** + * Append a double value. + * @param d A double. + * @return this + * @throws JSONException If the number is not finite. + */ + public JSONWriter value(double d) throws JSONException { + return this.value(new Double(d)); + } + + /** + * Append a long value. + * @param l A long. + * @return this + * @throws JSONException + */ + public JSONWriter value(long l) throws JSONException { + return this.append(Long.toString(l)); + } + + + /** + * Append an object value. + * @param o The object to append. It can be null, or a Boolean, Number, + * String, JSONObject, or JSONArray, or an object with a toJSONString() + * method. + * @return this + * @throws JSONException If the value is out of sequence. + */ + public JSONWriter value(Object o) throws JSONException { + return this.append(JSONObject.valueToString(o)); + } +} diff --git a/commons/src/main/java/org/json/XML.java b/commons/src/main/java/org/json/XML.java new file mode 100644 index 00000000..7e887216 --- /dev/null +++ b/commons/src/main/java/org/json/XML.java @@ -0,0 +1,405 @@ +package org.json; + +/* +Copyright (c) 2002 JSON.org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +The Software shall be used for Good, not Evil. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +import java.util.Iterator; + + +/** + * This provides static methods to convert an XML text into a JSONObject, + * and to covert a JSONObject into an XML text. + * @author JSON.org + * @version 2 + */ +@SuppressWarnings("unchecked") +public class XML { + + /** The Character '&'. */ + public static final Character AMP = new Character('&'); + + /** The Character '''. */ + public static final Character APOS = new Character('\''); + + /** The Character '!'. */ + public static final Character BANG = new Character('!'); + + /** The Character '='. */ + public static final Character EQ = new Character('='); + + /** The Character '>'. */ + public static final Character GT = new Character('>'); + + /** The Character '<'. */ + public static final Character LT = new Character('<'); + + /** The Character '?'. */ + public static final Character QUEST = new Character('?'); + + /** The Character '"'. */ + public static final Character QUOT = new Character('"'); + + /** The Character '/'. */ + public static final Character SLASH = new Character('/'); + + /** + * Replace special characters with XML escapes: + * 

+     * & (ampersand) is replaced by &amp;
+     * < (less than) is replaced by &lt;
+     * > (greater than) is replaced by &gt;
+     * " (double quote) is replaced by &quot;
+     *
+ * @param string The string to be escaped. + * @return The escaped string. + */ + public static String escape(String string) { + StringBuffer sb = new StringBuffer(); + for (int i = 0, len = string.length(); i < len; i++) { + char c = string.charAt(i); + switch (c) { + case '&': + sb.append("&"); + break; + case '<': + sb.append("<"); + break; + case '>': + sb.append(">"); + break; + case '"': + sb.append("""); + break; + default: + sb.append(c); + } + } + return sb.toString(); + } + + /** + * Scan the content following the named tag, attaching it to the context. + * @param x The XMLTokener containing the source string. + * @param context The JSONObject that will include the new material. + * @param name The tag name. + * @return true if the close tag is processed. + * @throws JSONException + */ + private static boolean parse(XMLTokener x, JSONObject context, + String name) throws JSONException { + char c; + int i; + String n; + JSONObject o = null; + String s; + Object t; + +// Test for and skip past these forms: +// +// +// +// +// Report errors for these forms: +// <> +// <= +// << + + t = x.nextToken(); + +// "); + return false; + } + x.back(); + } else if (c == '[') { + t = x.nextToken(); + if (t.equals("CDATA")) { + if (x.next() == '[') { + s = x.nextCDATA(); + if (s.length() > 0) { + context.accumulate("content", s); + } + return false; + } + } + throw x.syntaxError("Expected 'CDATA['"); + } + i = 1; + do { + t = x.nextMeta(); + if (t == null) { + throw x.syntaxError("Missing '>' after ' 0); + return false; + } else if (t == QUEST) { + +// "); + return false; + } else if (t == SLASH) { + +// Close tag + + } else if (t == SLASH) { + if (x.nextToken() != GT) { + throw x.syntaxError("Misshaped tag"); + } + context.accumulate(n, o); + return false; + +// Content, between <...> and + + } else if (t == GT) { + for (;;) { + t = x.nextContent(); + if (t == null) { + if (n != null) { + throw x.syntaxError("Unclosed tag " + n); + } + return false; + } else if (t instanceof String) { + s = (String)t; + if (s.length() > 0) { + o.accumulate("content", s); + } + +// Nested element + + } else if (t == LT) { + if (parse(x, o, n)) { + if (o.length() == 0) { + context.accumulate(n, ""); + } else if (o.length() == 1 && + o.opt("content") != null) { + context.accumulate(n, o.opt("content")); + } else { + context.accumulate(n, o); + } + return false; + } + } + } + } else { + throw x.syntaxError("Misshaped tag"); + } + } + } + } + + + /** + * Convert a well-formed (but not necessarily valid) XML string into a + * JSONObject. Some information may be lost in this transformation + * because JSON is a data format and XML is a document format. XML uses + * elements, attributes, and content text, while JSON uses unordered + * collections of name/value pairs and arrays of values. JSON does not + * does not like to distinguish between elements and attributes. + * Sequences of similar elements are represented as JSONArrays. Content + * text may be placed in a "content" member. Comments, prologs, DTDs, and + * <[ [ ]]> are ignored. + * @param string The source string. + * @return A JSONObject containing the structured data from the XML string. + * @throws JSONException + */ + public static JSONObject toJSONObject(String string) throws JSONException { + JSONObject o = new JSONObject(); + XMLTokener x = new XMLTokener(string); + while (x.more() && x.skipPast("<")) { + parse(x, o, null); + } + return o; + } + + + /** + * Convert a JSONObject into a well-formed, element-normal XML string. + * @param o A JSONObject. + * @return A string. + * @throws JSONException + */ + public static String toString(Object o) throws JSONException { + return toString(o, null); + } + + + /** + * Convert a JSONObject into a well-formed, element-normal XML string. + * @param o A JSONObject. + * @param tagName The optional name of the enclosing tag. + * @return A string. + * @throws JSONException + */ + public static String toString(Object o, String tagName) + throws JSONException { + StringBuffer b = new StringBuffer(); + int i; + JSONArray ja; + JSONObject jo; + String k; + Iterator keys; + int len; + String s; + Object v; + if (o instanceof JSONObject) { + +// Emit + + if (tagName != null) { + b.append('<'); + b.append(tagName); + b.append('>'); + } + +// Loop thru the keys. + + jo = (JSONObject)o; + keys = jo.keys(); + while (keys.hasNext()) { + k = keys.next().toString(); + v = jo.get(k); + if (v instanceof String) { + s = (String)v; + } else { + s = null; + } + +// Emit content in body + + if (k.equals("content")) { + if (v instanceof JSONArray) { + ja = (JSONArray)v; + len = ja.length(); + for (i = 0; i < len; i += 1) { + if (i > 0) { + b.append('\n'); + } + b.append(escape(ja.get(i).toString())); + } + } else { + b.append(escape(v.toString())); + } + +// Emit an array of similar keys + + } else if (v instanceof JSONArray) { + ja = (JSONArray)v; + len = ja.length(); + for (i = 0; i < len; i += 1) { + b.append(toString(ja.get(i), k)); + } + } else if (v.equals("")) { + b.append('<'); + b.append(k); + b.append("/>"); + +// Emit a new tag + + } else { + b.append(toString(v, k)); + } + } + if (tagName != null) { + +// Emit the close tag + + b.append("'); + } + return b.toString(); + +// XML does not have good support for arrays. If an array appears in a place +// where XML is lacking, synthesize an element. + + } else if (o instanceof JSONArray) { + ja = (JSONArray)o; + len = ja.length(); + for (i = 0; i < len; ++i) { + b.append(toString( + ja.opt(i), (tagName == null) ? "array" : tagName)); + } + return b.toString(); + } else { + s = (o == null) ? "null" : escape(o.toString()); + return (tagName == null) ? "\"" + s + "\"" : + (s.length() == 0) ? "<" + tagName + "/>" : + "<" + tagName + ">" + s + ""; + } + } +} \ No newline at end of file diff --git a/commons/src/main/java/org/json/XMLTokener.java b/commons/src/main/java/org/json/XMLTokener.java new file mode 100644 index 00000000..26faf35a --- /dev/null +++ b/commons/src/main/java/org/json/XMLTokener.java @@ -0,0 +1,293 @@ +package org.json; + +/* +Copyright (c) 2002 JSON.org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +The Software shall be used for Good, not Evil. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +/** + * The XMLTokener extends the JSONTokener to provide additional methods + * for the parsing of XML texts. + * @author JSON.org + * @version 2 + */ +@SuppressWarnings("unchecked") +public class XMLTokener extends JSONTokener { + + + /** The table of entity values. It initially contains Character values for + * amp, apos, gt, lt, quot. + */ + public static final java.util.HashMap entity; + + static { + entity = new java.util.HashMap(8); + entity.put("amp", XML.AMP); + entity.put("apos", XML.APOS); + entity.put("gt", XML.GT); + entity.put("lt", XML.LT); + entity.put("quot", XML.QUOT); + } + + /** + * Construct an XMLTokener from a string. + * @param s A source string. + */ + public XMLTokener(String s) { + super(s); + } + + /** + * Get the text in the CDATA block. + * @return The string up to the ]]>. + * @throws JSONException If the ]]> is not found. + */ + public String nextCDATA() throws JSONException { + char c; + int i; + StringBuffer sb = new StringBuffer(); + for (;;) { + c = next(); + if (c == 0) { + throw syntaxError("Unclosed CDATA"); + } + sb.append(c); + i = sb.length() - 3; + if (i >= 0 && sb.charAt(i) == ']' && + sb.charAt(i + 1) == ']' && sb.charAt(i + 2) == '>') { + sb.setLength(i); + return sb.toString(); + } + } + } + + + /** + * Get the next XML outer token, trimming whitespace. There are two kinds + * of tokens: the '<' character which begins a markup tag, and the content + * text between markup tags. + * + * @return A string, or a '<' Character, or null if there is no more + * source text. + * @throws JSONException + */ + public Object nextContent() throws JSONException { + char c; + StringBuffer sb; + do { + c = next(); + } while (Character.isWhitespace(c)); + if (c == 0) { + return null; + } + if (c == '<') { + return XML.LT; + } + sb = new StringBuffer(); + for (;;) { + if (c == '<' || c == 0) { + back(); + return sb.toString().trim(); + } + if (c == '&') { + sb.append(nextEntity(c)); + } else { + sb.append(c); + } + c = next(); + } + } + + + /** + * Return the next entity. These entities are translated to Characters: + * & ' > < ". + * @param a An ampersand character. + * @return A Character or an entity String if the entity is not recognized. + * @throws JSONException If missing ';' in XML entity. + */ + public Object nextEntity(char a) throws JSONException { + StringBuffer sb = new StringBuffer(); + for (;;) { + char c = next(); + if (Character.isLetterOrDigit(c) || c == '#') { + sb.append(Character.toLowerCase(c)); + } else if (c == ';') { + break; + } else { + throw syntaxError("Missing ';' in XML entity: &" + sb); + } + } + String s = sb.toString(); + Object e = entity.get(s); + return e != null ? e : a + s + ";"; + } + + + /** + * Returns the next XML meta token. This is used for skipping over + * and structures. + * @return Syntax characters (< > / = ! ?) are returned as + * Character, and strings and names are returned as Boolean. We don't care + * what the values actually are. + * @throws JSONException If a string is not properly closed or if the XML + * is badly structured. + */ + public Object nextMeta() throws JSONException { + char c; + char q; + do { + c = next(); + } while (Character.isWhitespace(c)); + switch (c) { + case 0: + throw syntaxError("Misshaped meta tag"); + case '<': + return XML.LT; + case '>': + return XML.GT; + case '/': + return XML.SLASH; + case '=': + return XML.EQ; + case '!': + return XML.BANG; + case '?': + return XML.QUEST; + case '"': + case '\'': + q = c; + for (;;) { + c = next(); + if (c == 0) { + throw syntaxError("Unterminated string"); + } + if (c == q) { + return Boolean.TRUE; + } + } + default: + for (;;) { + c = next(); + if (Character.isWhitespace(c)) { + return Boolean.TRUE; + } + switch (c) { + case 0: + case '<': + case '>': + case '/': + case '=': + case '!': + case '?': + case '"': + case '\'': + back(); + return Boolean.TRUE; + } + } + } + } + + + /** + * Get the next XML Token. These tokens are found inside of angle + * brackets. It may be one of these characters: / > = ! ? or it + * may be a string wrapped in single quotes or double quotes, or it may be a + * name. + * @return a String or a Character. + * @throws JSONException If the XML is not well formed. + */ + public Object nextToken() throws JSONException { + char c; + char q; + StringBuffer sb; + do { + c = next(); + } while (Character.isWhitespace(c)); + switch (c) { + case 0: + throw syntaxError("Misshaped element"); + case '<': + throw syntaxError("Misplaced '<'"); + case '>': + return XML.GT; + case '/': + return XML.SLASH; + case '=': + return XML.EQ; + case '!': + return XML.BANG; + case '?': + return XML.QUEST; + +// Quoted string + + case '"': + case '\'': + q = c; + sb = new StringBuffer(); + for (;;) { + c = next(); + if (c == 0) { + throw syntaxError("Unterminated string"); + } + if (c == q) { + return sb.toString(); + } + if (c == '&') { + sb.append(nextEntity(c)); + } else { + sb.append(c); + } + } + default: + +// Name + + sb = new StringBuffer(); + for (;;) { + sb.append(c); + c = next(); + if (Character.isWhitespace(c)) { + return sb.toString(); + } + switch (c) { + case 0: + case '>': + case '/': + case '=': + case '!': + case '?': + case '[': + case ']': + back(); + return sb.toString(); + case '<': + case '"': + case '\'': + throw syntaxError("Bad character in a name"); + } + } + } + } +} diff --git a/commons/src/main/java/st/ata/util/FPGenerator.java b/commons/src/main/java/st/ata/util/FPGenerator.java new file mode 100644 index 00000000..d759214b --- /dev/null +++ b/commons/src/main/java/st/ata/util/FPGenerator.java @@ -0,0 +1,451 @@ + +package st.ata.util; + +import java.util.Hashtable; + +/** + +

This class provides methods that construct fingerprints of strings +of bytes via operations in GF[2^d] for 0 < d <= 64. +GF[2^d] is represented as the set of polynomials of degree +d with coefficients in Z(2), modulo an irreducible +polynomial P of degree d. The representation of +polynomials is as an unsigned binary number in which the least +significant exponent is kept in the most significant bit. + +

Let S be a string of bytes and g(S) the string obtained by +taking the byte 0x01 followed by eight 0x00 +bytes followed by S. Let f(S) be the polynomial +associated to the string S viewed as a polynomial with +coefficients in the field Z(2). The fingerprint of S is simply +the value f(g(S)) modulo P. Because polynomials are +represented with the least significant coefficient in the most +significant bit, fingerprints of degree d are stored in the +d most significant bits of a long word. + +

Fingerprints can be used as a probably unique id for the input +string. More precisely, if P is chosen at random among +irreducible polynomials of degree d, then the probability that +any two strings A and B have the same fingerprint is +less than max(|A|,|B|)/2^(d+1) where |A| is the length +of A in bits. + +

The routines named extend[8] and fp[8] +return reduced results, while extend_[byte/char/int/long] +do not. An unreduced result is a number that is equal (mod +polynomial to the desired fingerprint but may have +degree degree or higher. The method reduce +reduces such a result to a polynomial of degree less than +degree. Obtaining reduced results takes longer than +obtaining unreduced results; thus, when fingerprinting long strings, +it's better to obtain irreduced results inside the fingerprinting loop +and use reduce to reduce to a fingerprint after the loop. + +*/ + +// Tested by: TestFPGenerator +@SuppressWarnings("unchecked") +public final class FPGenerator { + + /** Return a fingerprint generator. The fingerprints generated + will have degree degree and will be generated by + polynomial. If a generator based on + polynomial has already been created, it will be + returned. Requires that polynomial is an + irreducible polynomial of degree degree (the + array polynomials contains some irreducible + polynomials). */ + public static FPGenerator make(long polynomial, int degree) { + Long l = new Long(polynomial); + FPGenerator fpgen = (FPGenerator) generators.get(l); + if (fpgen == null) { + fpgen = new FPGenerator(polynomial, degree); + generators.put(l, fpgen); + } + return fpgen; + } + private static final Hashtable generators = new Hashtable(10); + + private static final long zero = 0; + private static final long one = 0x8000000000000000L; + + + /** Return a value equal (mod polynomial) to + fp and of degree less than degree. */ + public long reduce(long fp) { + int N = (8 - degree/8); + long local = (N == 8 ? 0 : fp & (-1L << 8*N)); + long temp = zero; + for (int i = 0; i < N; i++) { + temp ^= ByteModTable[8+i][((int)fp) & 0xff]; + fp >>>= 8; + }; + return local ^ temp; + } + + /** Extends f with lower eight bits of v + without full reduction. In other words, returns a + polynomial that is equal (mod polynomial) to the + desired fingerprint but may be of higher degree than the + desired fingerprint. */ + public long extend_byte(long f, int v) { + f ^= (0xff & v); + int i = (int)f; + long result = (f>>>8); + result ^= ByteModTable[7][i & 0xff]; + return result; + } + + /** Extends f with lower sixteen bits of v. + Does not reduce. */ + public long extend_char(long f, int v) { + f ^= (0xffff & v); + int i = (int)f; + long result = (f>>>16); + result ^= ByteModTable[6][i & 0xff]; i >>>= 8; + result ^= ByteModTable[7][i & 0xff]; + return result; + } + + /** Extends f with (all bits of) v. + Does not reduce. */ + public long extend_int(long f, int v) { + f ^= (0xffffffffL & v); + int i = (int)f; + long result = (f>>>32); + result ^= ByteModTable[4][i & 0xff]; i >>>= 8; + result ^= ByteModTable[5][i & 0xff]; i >>>= 8; + result ^= ByteModTable[6][i & 0xff]; i >>>= 8; + result ^= ByteModTable[7][i & 0xff]; + return result; + } + + /** Extends f with v. + Does not reduce. */ + public long extend_long(long f, long v) { + f ^= v; + long result = ByteModTable[0][(int)(f & 0xff)]; f >>>= 8; + result ^= ByteModTable[1][(int)(f & 0xff)]; f >>>= 8; + result ^= ByteModTable[2][(int)(f & 0xff)]; f >>>= 8; + result ^= ByteModTable[3][(int)(f & 0xff)]; f >>>= 8; + result ^= ByteModTable[4][(int)(f & 0xff)]; f >>>= 8; + result ^= ByteModTable[5][(int)(f & 0xff)]; f >>>= 8; + result ^= ByteModTable[6][(int)(f & 0xff)]; f >>>= 8; + result ^= ByteModTable[7][(int)(f & 0xff)]; + return result; + } + + + /** Compute fingerprint of "n" bytes of "buf" starting from + "buf[start]". Requires "[start, start+n)" is in bounds. */ + public long fp(byte[] buf, int start, int n) { + return extend(empty, buf, start, n); + } + + /** Compute fingerprint of (all bits of) "n" characters of "buf" + starting from "buf[i]". Requires "[i, i+n)" is in bounds. */ + public long fp(char[] buf, int start, int n) { + return extend(empty, buf, start, n); + } + +// COMMENTED OUT TO REMOVE Dependency on st.ata.util.Text +// /** Compute fingerprint of (all bits of) t */ +// public long fp(Text t) { +// return extend(empty, t); +// } + /** Compute fingerprint of (all bits of) the characters of "s". */ + public long fp(CharSequence s) { + return extend(empty, s); + } + + /** Compute fingerprint of (all bits of) "n" characters of "buf" + starting from "buf[i]". Requires "[i, i+n)" is in bounds. */ + public long fp(int[] buf, int start, int n) { + return extend(empty, buf, start, n); + } + + /** Compute fingerprint of (all bits of) "n" characters of "buf" + starting from "buf[i]". Requires "[i, i+n)" is in bounds. */ + public long fp(long[] buf, int start, int n) { + return extend(empty, buf, start, n); + } + + /** Compute fingerprint of the lower eight bits of the characters + of "s". */ + public long fp8(String s) { + return extend8(empty, s); + } + + /** Compute fingerprint of the lower eight bits of "n" characters + of "buf" starting from "buf[i]". Requires "[i, i+n)" is in + bounds. */ + public long fp8(char[] buf, int start, int n) { + return extend8(empty, buf, start, n); + } + + + /** Extends fingerprint f by adding the low eight + bits of "b". */ + public long extend(long f, byte v) { + return reduce(extend_byte(f, v)); + } + + /** Extends fingerprint f by adding (all bits of) + "v". */ + public long extend(long f, char v) { + return reduce(extend_char(f, v)); + } + + /** Extends fingerprint f by adding (all bits of) + "v". */ + public long extend(long f, int v) { + return reduce(extend_int(f, v)); + } + + /** Extends fingerprint f by adding (all bits of) + "v". */ + public long extend(long f, long v) { + return reduce(extend_long(f, v)); + } + + /** Extends fingerprint f by adding "n" bytes of + "buf" starting from "buf[start]". + Result is reduced. + Requires "[i, i+n)" is in bounds. */ + public long extend(long f, byte[] buf, int start, int n) { + for (int i = 0; i < n; i++) { + f = extend_byte(f, buf[start+i]); + } + return reduce(f); + } + + /** Extends fingerprint f by adding (all bits of) "n" + characters of "buf" starting from "buf[i]". + Result is reduced. + Requires "[i, i+n)" is in bounds. */ + public long extend(long f, char[] buf, int start, int n) { + for (int i = 0; i < n; i++) { + f = extend_char(f, buf[start+i]); + } + return reduce(f); + } + + /** Extends fingerprint f by adding (all bits of) + the characters of "s". + Result is reduced. */ + public long extend(long f, CharSequence s) { + int n = s.length(); + for (int i = 0; i < n; i++) { + int v = (int) s.charAt(i); + f = extend_char(f, v); + } + return reduce(f); + } + +// COMMENTED OUT TO REMOVE Dependency on st.ata.util.Text +// /** Extends fingerprint f by adding (all bits of) +// * t */ +// public long extend(long f, Text t) { +// return extend(f, t.buf, t.start, t.length()); +// } + + + /** Extends fingerprint f by adding (all bits of) "n" + characters of "buf" starting from "buf[i]". + Result is reduced. + Requires "[i, i+n)" is in bounds. */ + public long extend(long f, int[] buf, int start, int n) { + for (int i = 0; i < n; i++) { + f = extend_int(f, buf[start+i]); + } + return reduce(f); + } + + /** Extends fingerprint f by adding (all bits of) "n" + characters of "buf" starting from "buf[i]". + Result is reduced. + Requires "[i, i+n)" is in bounds. */ + public long extend(long f, long[] buf, int start, int n) { + for (int i = 0; i < n; i++) { + f = extend_long(f, buf[start+i]); + } + return reduce(f); + } + + /** Extends fingerprint f by adding the lower eight + bits of the characters of "s". + Result is reduced. */ + public long extend8(long f, String s) { + int n = s.length(); + for (int i = 0; i < n; i++) { + int x = (int) s.charAt(i); + f = extend_byte(f, x); + } + return reduce(f); + } + + /** Extends fingerprint f by adding the lower eight + bits of "n" characters of "buf" starting from "buf[i]". + Result is reduced. + Requires "[i, i+n)" is in bounds. */ + public long extend8(long f, char[] buf, int start, int n) { + for (int i = 0; i < n; i++) { + f = extend_byte(f, buf[start+i]); + } + return reduce(f); + } + + + /** Fingerprint of the empty string of bytes. */ + public final long empty; + + /** The number of bits in fingerprints generated by + this. */ + public final int degree; + + /** The polynomial used by this to generate + fingerprints. */ + public long polynomial; + + /** Result of reducing certain polynomials. Specifically, if + f(S) is bit string S interpreted as + a polynomial, f(ByteModTable[i][j]) equals + mod(x^(127 - 8*i) * f(j), P). */ + private long[][] ByteModTable; + + /** Create a fingerprint generator. The fingerprints generated + will have degree degree and will be generated by + polynomial. Requires that + polynomial is an irreducible polynomial of degree + degree (the array polynomials + contains some irreducible polynomials). */ + private FPGenerator(long polynomial, int degree) { + this.degree = degree; + this.polynomial = polynomial; + ByteModTable = new long[16][256]; + + long[] PowerTable = new long[128]; + + long x_to_the_i = one; + long x_to_the_degree_minus_one = (one >>> (degree-1)); + for (int i = 0; i < 128; i++) { + // Invariants: + // x_to_the_i = mod(x^i, polynomial) + // forall 0 <= j < i, PowerTable[i] = mod(x^i, polynomial) + PowerTable[i] = x_to_the_i; + boolean overflow = ((x_to_the_i & x_to_the_degree_minus_one) != 0); + x_to_the_i >>>= 1; + if (overflow) { + x_to_the_i ^= polynomial; + } + } + this.empty = PowerTable[64]; + + for (int i = 0; i < 16; i++) { + // Invariant: forall 0 <= i' < i, forall 0 <= j' < 256, + // ByteModTable[i'][j'] = mod(x^(127 - 8*i') * f(j'), polynomial) + for (int j = 0; j < 256; j++) { + // Invariant: forall 0 <= i' < i, forall 0 <= j' < j, + // ByteModTable[i'][j'] = mod(x^(degree+i')*f(j'),polynomial) + long v = zero; + for (int k = 0; k < 8; k++) { + // Invariant: + // v = mod(x^(degree+i) * f(j & ((1<d between 1 and 64 (inclusive), + polynomials[d][i] is an irreducible polynomial of + degree d. There are at least two irreducible + polynomials for each degree. */ + public static final long polynomials[][] = { + null, + {0xC000000000000000L, 0xC000000000000000L}, + {0xE000000000000000L, 0xE000000000000000L}, + {0xD000000000000000L, 0xB000000000000000L}, + {0xF800000000000000L, 0xF800000000000000L}, + {0xEC00000000000000L, 0xBC00000000000000L}, + {0xDA00000000000000L, 0xB600000000000000L}, + {0xE500000000000000L, 0xE500000000000000L}, + {0x9680000000000000L, 0xD480000000000000L}, + {0x80C0000000000000L, 0x8840000000000000L}, + {0xB0A0000000000000L, 0xE9A0000000000000L}, + {0xD9F0000000000000L, 0xC9B0000000000000L}, + {0xE758000000000000L, 0xDE98000000000000L}, + {0xE42C000000000000L, 0x94E4000000000000L}, + {0xD4CE000000000000L, 0xB892000000000000L}, + {0xE2AB000000000000L, 0x9E39000000000000L}, + {0xCCE4800000000000L, 0x9783800000000000L}, + {0xD8F8C00000000000L, 0xA9CDC00000000000L}, + {0x9A28200000000000L, 0xFD79E00000000000L}, + {0xC782500000000000L, 0x96CD300000000000L}, + {0xBEE6880000000000L, 0xE902C80000000000L}, + {0x86D7E40000000000L, 0xF066340000000000L}, + {0x9888060000000000L, 0x910ABE0000000000L}, + {0xFF30E30000000000L, 0xB27EFB0000000000L}, + {0x8E375B8000000000L, 0xA03D948000000000L}, + {0xD1415C4000000000L, 0xF5357CC000000000L}, + {0x91A916E000000000L, 0xB6CE66E000000000L}, + {0xE6D2FC5000000000L, 0xD55882B000000000L}, + {0x9A3BA0B800000000L, 0xFBD654E800000000L}, + {0xAEA5D2E400000000L, 0xF0E533AC00000000L}, + {0xDA88B7BE00000000L, 0xAA3AAEDE00000000L}, + {0xBA75BB4300000000L, 0xF5A811C500000000L}, + {0x9B6C9A2F80000000L, 0x9603CCED80000000L}, + {0xFABB538840000000L, 0xE2747090C0000000L}, + {0x8358898EA0000000L, 0x8C698D3D20000000L}, + {0xDA8ABD5BF0000000L, 0xF6DF3A0AF0000000L}, + {0xB090C3F758000000L, 0xD3B4D3D298000000L}, + {0xAD9882F5BC000000L, 0x88DA4FB544000000L}, + {0xC3C366272A000000L, 0xDCCF2A2262000000L}, + {0x9BC0224A97000000L, 0xAF5D96F273000000L}, + {0x8643FFF621800000L, 0x8E390C6EDC800000L}, + {0xE45C01919BC00000L, 0xCBB34C8945C00000L}, + {0x80D8141BC2E00000L, 0x886AFC3912200000L}, + {0xF605807C26500000L, 0xA3B92D28F6300000L}, + {0xCE9A2CFC76280000L, 0x98400C1921280000L}, + {0xF61894904C040000L, 0xC8BE6DBCEC8C0000L}, + {0xE3A44C104D160000L, 0xCA84A59443760000L}, + {0xC7E84953A11B0000L, 0xD9D4F6AA02CB0000L}, + {0xC26CDD1C9A358000L, 0x8BE8478434328000L}, + {0xAE125DBEB198C000L, 0xFCC5DBFD5AAAC000L}, + {0x86DE52A079A6A000L, 0xC5F16BD883816000L}, + {0xDF82486AAFE37000L, 0xA293EC735692D000L}, + {0xE91ABA275C272800L, 0xD686192874E3C800L}, + {0x963D0960DAB3FC00L, 0xBA9DE62072621400L}, + {0xA2188C4E8A46CE00L, 0xD31F75BCB4977E00L}, + {0xC43A416020A6CB00L, 0x99F57FECA12B3900L}, + {0xA4F72EF82A58AE80L, 0xCECE4391B81DA380L}, + {0xB39F119264BC0940L, 0x80A277D20DABB9C0L}, + {0xFD6616C0CBFA0B20L, 0xED16E64117DC11A0L}, + {0xFFA8BC44327B5390L, 0xEDFB13DB3B66C210L}, + {0xCAE8EB99E73AB548L, 0xC86135B6EA2F0B98L}, + {0xBA49BADCDD19B16CL, 0x8F1944AFB18564C4L}, + {0xECFC86D765EABBEEL, 0x9190E1C46CC13702L}, + {0xE1F8D6B3195D6D97L, 0xDF70267FF5E4C979L}, + {0xD74307D3FD3382DBL, 0x9999B3FFDC769B48L} + }; + + /** The standard 64-bit fingerprint generator using + polynomials[0][64]. */ + public static final FPGenerator std64 = make(polynomials[64][0], 64); + + /** A standard 32-bit fingerprint generator using + polynomials[0][32]. */ + public static final FPGenerator std32 = make(polynomials[32][0], 32); + + // Below added by St.Ack on 09/30/2004. + /** A standard 40-bit fingerprint generator using + polynomials[0][40]. */ + public static final FPGenerator std40 = make(polynomials[40][0], 40); + /** A standard 24-bit fingerprint generator using + polynomials[0][24]. */ + public static final FPGenerator std24 = make(polynomials[24][0], 24); +} diff --git a/commons/src/main/resources/effective_tld_names.dat b/commons/src/main/resources/effective_tld_names.dat new file mode 100644 index 00000000..be84f22a --- /dev/null +++ b/commons/src/main/resources/effective_tld_names.dat @@ -0,0 +1,3030 @@ +// downloaded from: +// http://lxr.mozilla.org/mozilla/source/netwerk/dns/src/effective_tld_names.dat?raw=1 +// on 2007-06-13 + +// ac : http://en.wikipedia.org/wiki/.am +ac +com.ac +edu.ac +gov.ac +net.ac +mil.ac +org.ac + +// ad : http://en.wikipedia.org/wiki/.ad +ad +nom.ad + +// ae : http://en.wikipedia.org/wiki/.ae +ae +net.ae +gov.ae +ac.ae +sch.ae +org.ae +mil.ae +pro.ae +name.ae + +// aero : see http://www.information.aero/index.php?id=66 +aero +accident-investigation.aero +accident-prevention.aero +aerobatic.aero +aeroclub.aero +aerodrome.aero +agents.aero +aircraft.aero +airline.aero +airport.aero +air-surveillance.aero +airtraffic.aero +air-traffic-control.aero +ambulance.aero +amusement.aero +association.aero +author.aero +ballooning.aero +broker.aero +caa.aero +cargo.aero +catering.aero +certification.aero +championship.aero +charter.aero +civilaviation.aero +club.aero +conference.aero +consultant.aero +consulting.aero +control.aero +council.aero +crew.aero +design.aero +dgca.aero +educator.aero +emergency.aero +engine.aero +engineer.aero +entertainment.aero +equipment.aero +exchange.aero +express.aero +federation.aero +flight.aero +freight.aero +fuel.aero +gliding.aero +government.aero +groundhandling.aero +group.aero +hanggliding.aero +homebuilt.aero +insurance.aero +journal.aero +journalist.aero +leasing.aero +logistics.aero +magazine.aero +maintenance.aero +marketplace.aero +media.aero +microlight.aero +modelling.aero +navigation.aero +parachuting.aero +paragliding.aero +passenger-association.aero +pilot.aero +press.aero +production.aero +recreation.aero +repbody.aero +res.aero +research.aero +rotorcraft.aero +safety.aero +scientist.aero +services.aero +show.aero +skydiving.aero +software.aero +student.aero +taxi.aero +trader.aero +trading.aero +trainer.aero +union.aero +workinggroup.aero +works.aero + +// af : http://www.nic.af/help.jsp +af +gov.af +com.af +org.af +net.af +edu.af + +// ag : http://www.nic.ag/prices.htm +ag +com.ag +org.ag +net.ag +co.ag +nom.ag + +// ai : http://nic.com.ai/ +ai +off.ai +com.ai +net.ai +org.ai + +// al : http://www.inima.al/Domains.html +gov.al +edu.al +org.al +com.al +net.al + +// am : http://en.wikipedia.org/wiki/.am +am + +// an : http://www.una.an/an_domreg/default.asp +an +com.an +net.an +org.an +edu.an + +// ao : http://en.wikipedia.org/wiki/.ao +// list of 2nd level TLDs ? +ao + +// aq : http://en.wikipedia.org/wiki/.aq +aq + +// ar : http://en.wikipedia.org/wiki/.ar +*.ar +!congresodelalengua3.ar +!educ.ar +!gobiernoelectronico.ar +!mecon.ar +!nacion.ar +!nic.ar +!promocion.ar +!retina.ar +!uba.ar + +// arpa : http://en.wikipedia.org/wiki/.arpa +e164.arpa +in-addr.arpa +ip6.arpa +uri.arpa +urn.arpa + +// as : http://en.wikipedia.org/wiki/.as +as + +// at : http://en.wikipedia.org/wiki/.at +at +gv.at +ac.at +co.at +or.at + +// au : http://en.wikipedia.org/wiki/.au +*.au +// au geographical names (vic.au etc... are covered above) +act.edu.au +nsw.edu.au +nt.edu.au +qld.edu.au +sa.edu.au +tas.edu.au +vic.edu.au +wa.edu.au +act.gov.au +nsw.gov.au +nt.gov.au +qld.gov.au +sa.gov.au +tas.gov.au +vic.gov.au +wa.gov.au + +// aw : http://en.wikipedia.org/wiki/.aw +aw +com.aw + +// ax : http://en.wikipedia.org/wiki/.ax +ax + +// az : http://en.wikipedia.org/wiki/.az +az +com.az +net.az +int.az +gov.az +org.az +edu.az +info.az +pp.az +mil.az +name.az +biz.az + +// ba : http://en.wikipedia.org/wiki/.ba +ba +org.ba +net.ba +edu.ba +gov.ba +mil.ba +unsa.ba +unbi.ba +co.ba +com.ba +rs.ba + +// bb : http://en.wikipedia.org/wiki/.bb +bb +com.bb +edu.bb +gov.bb +net.bb +org.bb + +// bd : http://en.wikipedia.org/wiki/.bd +*.bd + +// be : http://en.wikipedia.org/wiki/.be +be +ac.be + +// bf : http://en.wikipedia.org/wiki/.bf +bf + +// bg : http://en.wikipedia.org/wiki/.bg +bg + +// bh : http://en.wikipedia.org/wiki/.bh +// list of 2nd level tlds ? +bh + +// bi : http://en.wikipedia.org/wiki/.bi +// list of 2nd level tlds ? +bi + +// biz : http://en.wikipedia.org/wiki/.biz +biz + +// bj : http://en.wikipedia.org/wiki/.bj +// list of 2nd level tlds ? +bj + +// bm : http://www.bermudanic.bm/dnr-text.txt +bm +com.bm +edu.bm +gov.bm +net.bm +org.bm + +// bn : http://en.wikipedia.org/wiki/.bn +*.bn + +// bo : http://www.nic.bo/ +bo +com.bo +edu.bo +gov.bo +gob.bo +int.bo +org.bo +net.bo +mil.bo +tv.bo + +// br : http://en.wikipedia.org/wiki/.br +*.br + +// bs : http://www.nic.bs/rules.html +bs +com.bs +net.bs +org.bs +edu.bs +gov.bs + +// bt : http://en.wikipedia.org/wiki/.bt +*.bt + +// bw : http://en.wikipedia.org/wiki/.bw +// list of 2nd level tlds ? +bw + +// by : http://en.wikipedia.org/wiki/.by +// list of 2nd level tlds ? +by + +// bz : http://en.wikipedia.org/wiki/.bz +// list of 2nd level tlds ? +bz + +// ca : http://en.wikipedia.org/wiki/.ca +ca +// ca geographical names +ab.ca +bc.ca +mb.ca +nb.ca +nf.ca +nl.ca +ns.ca +nt.ca +nu.ca +on.ca +pe.ca +qc.ca +sk.ca +yk.ca + +// cat : http://en.wikipedia.org/wiki/.cat +cat + +// cc : http://en.wikipedia.org/wiki/.cc +cc + +// cd : http://en.wikipedia.org/wiki/.cd +cd + +// cf : http://en.wikipedia.org/wiki/.cf +cf + +// cg : http://en.wikipedia.org/wiki/.cg +cg + +// ch : http://en.wikipedia.org/wiki/.ch +ch + +// ci : http://en.wikipedia.org/wiki/.ci +// list of 2nd level tlds ? +ci + +// ck : http://en.wikipedia.org/wiki/.ck +*.ck + +// cl : http://en.wikipedia.org/wiki/.cl +cl + +// cm : http://en.wikipedia.org/wiki/.cm +cm + +// cn : http://en.wikipedia.org/wiki/.cn +cn +ac.cn +com.cn +edu.cn +gov.cn +net.cn +org.cn +// cn geographic names +ah.cn +bj.cn +cq.cn +fj.cn +gd.cn +gs.cn +gz.cn +gx.cn +ha.cn +hb.cn +he.cn +hi.cn +hl.cn +hn.cn +jl.cn +js.cn +jx.cn +ln.cn +nm.cn +nx.cn +qh.cn +sc.cn +sd.cn +sh.cn +sn.cn +sx.cn +tj.cn +xj.cn +xz.cn +yn.cn +zj.cn + +// co : http://en.wikipedia.org/wiki/.co +*.co + +// com : http://en.wikipedia.org/wiki/.com +com + +// coop : http://en.wikipedia.org/wiki/.coop +coop + +// cr : http://en.wikipedia.org/wiki/.cr +*.cr + +// cu : http://en.wikipedia.org/wiki/.cu +cu +com.cu +edu.cu +org.cu +net.cu +gov.cu +inf.cu + +// cv : http://en.wikipedia.org/wiki/.cv +cv + +// cx : http://en.wikipedia.org/wiki/.cx +cx + +// cy : http://en.wikipedia.org/wiki/.cy +*.cy + +// cz : http://en.wikipedia.org/wiki/.cz +cz + +// de : http://en.wikipedia.org/wiki/.de +de + +// dj : http://en.wikipedia.org/wiki/.dj +dj + +// dk : http://en.wikipedia.org/wiki/.dk +dk + +// dm : http://en.wikipedia.org/wiki/.dm +dm +com.dm +net.dm +org.dm + +// do : http://en.wikipedia.org/wiki/.do +*.do + +// dz : http://en.wikipedia.org/wiki/.dz +dz +com.dz +org.dz +net.dz +gov.dz +edu.dz +asso.dz +pol.dz +art.dz + +// ec : http://www.nic.ec/reg/paso1.asp +ec +com.ec +info.ec +net.ec +fin.ec +med.ec +pro.ec +org.ec +edu.ec +gov.ec +mil.ec + +// edu : http://en.wikipedia.org/wiki/.edu +edu + +// ee : http://www3.eenet.ee/ee/application.html +ee +com.ee +org.ee +fie.ee +pri.ee + +// eg : http://en.wikipedia.org/wiki/.eg +*.eg + +// er : http://en.wikipedia.org/wiki/.er +*.er + +// es : https://www.nic.es/site_ingles/ingles/dominios/index.html +es +com.es +nom.es +org.es +gob.es +edu.es + +// et : http://en.wikipedia.org/wiki/.et +*.et + +// eu : http://en.wikipedia.org/wiki/.eu +eu + +// fi : http://en.wikipedia.org/wiki/.fi +fi + +// fj : http://en.wikipedia.org/wiki/.fj +*.fj + +// fk : http://en.wikipedia.org/wiki/.fk +*.fk + +// fm : http://en.wikipedia.org/wiki/.fm +fm + +// fo : http://en.wikipedia.org/wiki/.fo +fo + +// fr : http://www.afnic.fr/ +fr +// domaines descriptifs : http://www.afnic.fr/obtenir/chartes/nommage-fr/annexe-descriptifs +fr +com.fr +asso.fr +nom.fr +prd.fr +presse.fr +tm.fr +// domaines sectoriels : http://www.afnic.fr/obtenir/chartes/nommage-fr/annexe-sectoriels +aeroport.fr +assedic.fr +avocat.fr +avoues.fr +cci.fr +chambagri.fr +chirurgiens-dentistes.fr +experts-comptables.fr +geometre-expert.fr +gouv.fr +greta.fr +huissier-justice.fr +medecin.fr +notaires.fr +pharmacien.fr +port.fr +veterinaire.fr + +// ga : http://en.wikipedia.org/wiki/.ga +ga + +// gd : http://en.wikipedia.org/wiki/.gd +gd + +// ge : http://www.nic.net.ge/policy_en.pdf +ge +com.ge +edu.ge +gov.ge +org.ge +mil.ge +net.ge +pvt.ge + +// gf : http://en.wikipedia.org/wiki/.gf +gf + +// gg : http://www.channelisles.net/tandc.shtml +gg +co.gg +org.gg +net.gg +sch.gg +gov.gg + +// gh : http://www.ghana.com/domain.htm +*.gh + +// gi : http://www.nic.gi/rules.html +gi +com.gi +ltd.gi +gov.gi +mod.gi +edu.gi +org.gi + +// gl : http://en.wikipedia.org/wiki/.gl +gl + +// gm : http://www.nic.gm/htmlpages%5Cgm-policy.htm +gm + +// gn : http://psg.com/dns/gn/gn.txt +*.gn + +// gov : http://en.wikipedia.org/wiki/.gov +gov + +// gp : http://www.nic.gp/index_en.php?url=charte_en.php +gp +com.gp +net.gp +edu.gp +org.gp + +// gq : http://en.wikipedia.org/wiki/.gq +gq + +// gr : https://grweb.ics.forth.gr/english/1617-B-2002.html +gr +com.gr +edu.gr +net.gr +org.gr +gov.gr + +// gs : http://en.wikipedia.org/wiki/.gs +gs + +// gt : http://www.gt/politicas.html +*.gt + +// gu : http://gadao.gov.gu/registration.txt +*.gu + +// gw : http://en.wikipedia.org/wiki/.gw +gw + +// gy : http://en.wikipedia.org/wiki/.gy +gy + +// hk : http://en.wikipedia.org/wiki/.hk +hk +com.hk +edu.hk +gov.hk +idv.hk +net.hk +org.hk + +// hm : http://en.wikipedia.org/wiki/.hm +hm + +// hn : http://www.nic.hn/politicas/ps02,,05.html +hn +com.hn +edu.hn +org.hn +net.hn +mil.hn +gob.hn + +// hr : http://www.dns.hr/documents/pdf/HRTLD-regulations.pdf +hr +iz.hr +from.hr +name.hr +com.hr + +// ht : http://www.nic.ht/info/charte.cfm +ht +com.ht +shop.ht +firm.ht +info.ht +adult.ht +net.ht +pro.ht +org.ht +med.ht +art.ht +coop.ht +pol.ht +asso.ht +edu.ht +rel.ht +gouv.ht +perso.ht + +// hu : http://www.domain.hu/domain/English/sld.html +hu +co.hu +info.hu +org.hu +priv.hu +sport.hu +tm.hu +2000.hu +agrar.hu +bolt.hu +casino.hu +city.hu +erotica.hu +erotika.hu +film.hu +forum.hu +games.hu +hotel.hu +ingatlan.hu +jogasz.hu +konyvelo.hu +lakas.hu +media.hu +news.hu +reklam.hu +sex.hu +shop.hu +suli.hu +szex.hu +tozsde.hu +utazas.hu +video.hu + +// id : http://en.wikipedia.org/wiki/.id +*.id + +// ie : http://en.wikipedia.org/wiki/.ie +ie + +// il : http://en.wikipedia.org/wiki/.il +*.il + +// im : https://www.nic.im/pdfs/imfaqs.pdf +im +co.im +ltd.co.im +plc.co.im +net.im +gov.im +org.im +nic.im +ac.im + +// in : http://en.wikipedia.org/wiki/.in +in +co.in +firm.in +net.in +org.in +gen.in +ind.in +nic.in +ac.in +edu.in +res.in +gov.in +mil.in + +// info : http://en.wikipedia.org/wiki/.info +info + +// int : http://en.wikipedia.org/wiki/.int +int + +// io : http://www.nic.io/rules.html +// list of 2nd level tlds ? +io + +// iq : http://en.wikipedia.org/wiki/.iq +// no registrar website found, but google shows .gov.iq and .edu.iq websites +iq +gov.iq +edu.iq + +// ir : http://www.nic.ir/ascii/Appendix1.htm +ir +ac.ir +co.ir +gov.ir +id.ir +net.ir +org.ir +sch.ir + +// is : http://www.isnic.is/domain/rules.php +is +net.is +com.is +edu.is +gov.is +org.is +int.is + +// it : http://en.wikipedia.org/wiki/.it +it +gov.edu +// geo-names found at http://www.nic.it/RA/en/domini/regole/nomi-riservati.pdf +Agrigento.it +AG.it +Alessandria.it +AL.it +Ancona.it +AN.it +Aosta.it +Aoste.it +AO.it +Arezzo.it +AR.it +Ascoli-Piceno.it +AscoliPiceno.it +AP.it +Asti.it +AT.it +Avellino.it +AV.it +Bari.it +BA.it +BarlettaAndriaTrani.it +Barletta-Andria-Trani.it +Belluno.it +BL.it +Benevento.it +BN.it +Bergamo.it +BG.it +Biella.it +BI.it +Bologna.it +BO.it +Bolzano.it +Bozen.it +Balsan.it +Alto-Adige.it +AltoAdige.it +Suedtirol.it +BZ.it +Brescia.it +BS.it +Brindisi.it +BR.it +Cagliari.it +CA.it +Caltanissetta.it +CL.it +Campobasso.it +CB.it +Caserta.it +CE.it +Catania.it +CT.it +Catanzaro.it +CZ.it +Chieti.it +CH.it +Como.it +CO.it +Cosenza.it +CS.it +Cremona.it +CR.it +Crotone.it +KR.it +Cuneo.it +CN.it +Enna.it +EN.it +Fermo.it +Ferrara.it +FE.it +Firenze.it +Florence.it +FI.it +Foggia.it +FG.it +Forli-Cesena.it +ForliCesena.it +FC.it +Frosinone.it +FR.it +Genova.it +Genoa.it +GE.it +Gorizia.it +GO.it +Grosseto.it +GR.it +Imperia.it +IM.it +Isernia.it +IS.it +LAquila.it +Aquila.it +AQ.it +La-Spezia.it +LaSpezia.it +SP.it +Latina.it +LT.it +Lecce.it +LE.it +Lecco.it +LC.it +Livorno.it +LI.it +Lodi.it +LO.it +Lucca.it +LU.it +Macerata.it +MC.it +Mantova.it +MN.it +Massa-Carrara.it +MassaCarrara.it +MS.it +Matera.it +MT.it +Messina.it +ME.it +Milano.it +Milan.it +MI.it +Modena.it +MO.it +Monza.it +Napoli.it +Naples.it +NA.it +Novara.it +NO.it +Nuoro.it +NU.it +Oristano.it +OR.it +Padova.it +Padua.it +PD.it +Palermo.it +PA.it +Parma.it +PR.it +Pavia.it +PV.it +Perugia.it +PG.it +Pescara.it +PE.it +Pesaro-Urbino.it +PesaroUrbino.it +PU.it +Piacenza.it +PC.it +Pisa.it +PI.it +Pistoia.it +PT.it +Pordenone.it +PN.it +Potenza.it +PZ.it +Prato.it +PO.it +Ragusa.it +RG.it +Ravenna.it +RA.it +Reggio-Calabria.it +ReggioCalabria.it +RC.it +Reggio-Emilia.it +ReggioEmilia.it +RE.it +Rieti.it +RI.it +Rimini.it +RN.it +Roma.it +Rome.it +RM.it +Rovigo.it +RO.it +Salerno.it +SA.it +Sassari.it +SS.it +Savona.it +SV.it +Siena.it +SI.it +Siracusa.it +SR.it +Sondrio.it +SO.it +Taranto.it +TA.it +Teramo.it +TE.it +Terni.it +TR.it +Torino.it +Turin.it +TO.it +Trapani.it +TP.it +Trento.it +Trentino.it +TN.it +Treviso.it +TV.it +Trieste.it +TS.it +Udine.it +UD.it +Varese.it +VA.it +Venezia.it +Venice.it +VE.it +Verbania.it +VB.it +Vercelli.it +VC.it +Verona.it +VR.it +Vibo-Valentia.it +ViboValentia.it +VV.it +Vicenza.it +VI.it +Viterbo.it +VT.it + +// je : http://www.channelisles.net/tandc.shtml +je +co.je +org.je +net.je +sch.je +gov.je + +// jm : http://www.com.jm/register.html +*.jm + +// jo : http://www.nis.gov.jo/dns/reg.html +jo +com.jo +org.jo +net.jo +edu.jo +gov.jo +mil.jo +myname.jo + +// jobs : http://en.wikipedia.org/wiki/.jobs +jobs + +// jp : http://en.wikipedia.org/wiki/.jp +jp +ac.jp +ad.jp +co.jp +ed.jp +go.jp +gr.jp +lg.jp +ne.jp +or.jp +// jp geographical names +// I can't find an official english explanantrion, but used https://bugzilla.mozilla.org/show_bug.cgi?id=252342#c31 +*.aichi.jp +*.akita.jp +*.aomori.jp +*.chiba.jp +*.ehime.jp +*.fukui.jp +*.fukuoka.jp +*.fukushima.jp +*.gifu.jp +*.gunma.jp +*.hiroshima.jp +*.hokkaido.jp +*.hyogo.jp +*.ibaraki.jp +*.ishikawa.jp +*.iwate.jp +*.kagawa.jp +*.kagoshima.jp +*.kanagawa.jp +*.kawasaki.jp +*.kitakyushu.jp +*.kobe.jp +*.kochi.jp +*.kumamoto.jp +*.kyoto.jp +*.mie.jp +*.miyagi.jp +*.miyazaki.jp +*.nagano.jp +*.nagasaki.jp +*.nagoya.jp +*.nara.jp +*.niigata.jp +*.oita.jp +*.okayama.jp +*.okinawa.jp +*.osaka.jp +*.saga.jp +*.saitama.jp +*.sapporo.jp +*.sendai.jp +*.shiga.jp +*.shimane.jp +*.shizuoka.jp +*.tochigi.jp +*.tokushima.jp +*.tokyo.jp +*.tottori.jp +*.toyama.jp +*.wakayama.jp +*.yamagata.jp +*.yamaguchi.jp +*.yamanashi.jp +*.yokohama.jp +!metro.tokyo.jp +!pref.aichi.jp +!pref.akita.jp +!pref.aomori.jp +!pref.chiba.jp +!pref.ehime.jp +!pref.fukui.jp +!pref.fukuoka.jp +!pref.fukushima.jp +!pref.gifu.jp +!pref.gunma.jp +!pref.hiroshima.jp +!pref.hokkaido.jp +!pref.hyogo.jp +!pref.ibaraki.jp +!pref.ishikawa.jp +!pref.iwate.jp +!pref.kagawa.jp +!pref.kagoshima.jp +!pref.kanagawa.jp +!pref.kochi.jp +!pref.kumamoto.jp +!pref.kyoto.jp +!pref.mie.jp +!pref.miyagi.jp +!pref.miyazaki.jp +!pref.nagano.jp +!pref.nagasaki.jp +!pref.nara.jp +!pref.niigata.jp +!pref.oita.jp +!pref.okayama.jp +!pref.okinawa.jp +!pref.osaka.jp +!pref.saga.jp +!pref.saitama.jp +!pref.shiga.jp +!pref.shimane.jp +!pref.shizuoka.jp +!pref.tochigi.jp +!pref.tokushima.jp +!pref.tottori.jp +!pref.toyama.jp +!pref.wakayama.jp +!pref.yamagata.jp +!pref.yamaguchi.jp +!pref.yamanashi.jp +!city.chiba.jp +!city.fukuoka.jp +!city.hiroshima.jp +!city.kawasaki.jp +!city.kitakyushu.jp +!city.kobe.jp +!city.kyoto.jp +!city.nagoya.jp +!city.osaka.jp +!city.saitama.jp +!city.sapporo.jp +!city.sendai.jp +!city.shizuoka.jp +!city.yokohama.jp + +// ke : http://www.kenic.or.ke/index.php?option=com_content&task=view&id=117&Itemid=145 +*.ke + +// kg : http://www.domain.kg/dmn_n.html +kg +org.kg +net.kg +com.kg +edu.kg +gov.kg +mil.kg + +// kh : http://www.mptc.gov.kh/dns_registration.htm +*.kh + +// ki : http://www.ki/dns/index.html +ki +edu.ki +biz.ki +net.ki +org.ki +gov.ki +info.ki +com.ki + +// km : http://en.wikipedia.org/wiki/.km +km + +// kn : http://en.wikipedia.org/wiki/.kn +kn + +// kr : http://domain.nida.or.kr/eng/structure.jsp +kr +ac.kr +co.kr +go.kr +ne.kr +or.kr +re.kr +pe.kr +한글.kr utf8 ! + +// kw : http://en.wikipedia.org/wiki/.kw +*.kw + +// ky : http://www.icta.ky/da_ky_reg_dom.php +ky +edu.ky +gov.ky +com.ky +org.ky +net.ky + +// kz : http://en.wikipedia.org/wiki/.kz +kz +org.kz +edu.kz +net.kz +gov.kz +mil.kz +com.kz + +// la : http://en.wikipedia.org/wiki/.la +la + +// lb : http://en.wikipedia.org/wiki/.lb +*.lb + +// lc : http://en.wikipedia.org/wiki/.lc +lc +com.lc +org.lc +edu.lc +gov.lc + +// li : http://en.wikipedia.org/wiki/.li +li + +// lk : http://www.nic.lk/seclevpr.html +lk +gov.lk +sch.lk +net.lk +int.lk +com.lk +org.lk +edu.lk +ngo.lk +soc.lk +web.lk +ltd.lk +assn.lk +grp.lk +hotel.lk + +// lr : http://psg.com/dns/lr/lr.txt +*.lr + +// ls : http://en.wikipedia.org/wiki/.ls +ls +co.ls +org.ls + +// lt : http://en.wikipedia.org/wiki/.lt +lt + +// lu : http://www.dns.lu/en/ +lu + +// lv : http://www.nic.lv/DNS/En/generic.php +lv +com.lv +edu.lv +gov.lv +org.lv +mil.lv +id.lv +net.lv +asn.lv +conf.lv + +// ly : http://www.nic.ly/regulations.php +ly +com.ly +net.ly +gov.ly +plc.ly +edu.ly +sch.ly +med.ly +org.ly +id.ly + +// ma : http://en.wikipedia.org/wiki/.ma +// list of 2nd level tlds ? +ma +co.ma +net.ma +gov.ma +org.ma + +// mc : http://www.nic.mc/ +mc +tm.mc +asso.mc + +// md : http://en.wikipedia.org/wiki/.md +md + +// mg : http://www.nic.mg/tarif.htm +mg +org.mg +nom.mg +gov.mg +prd.mg +tm.mg +edu.mg +mil.mg +com.mg + +// mh : http://en.wikipedia.org/wiki/.mh +mh + +// mil : http://en.wikipedia.org/wiki/.mil +mil + +// mk : http://en.wikipedia.org/wiki/.mk +// list of 2nd level tlds ? +mk +com.mk +gov.mk +org.mk +net.mk +edu.mk + +// ml : http://www.gobin.info/domainname/ml-template.doc +*.ml + +// mm : http://en.wikipedia.org/wiki/.mm +*.mm + +// mn : http://en.wikipedia.org/wiki/.mn +mn +gov.mn +edu.mn +org.mn + +// mo : http://www.monic.net.mo/ +mo +com.mo +net.mo +org.mo +edu.mo +gov.mo + +// mobi : http://en.wikipedia.org/wiki/.mobi +mobi + +// mp : http://www.dot.mp/ +mp + +// mq : http://en.wikipedia.org/wiki/.mq +mq + +// mr : http://en.wikipedia.org/wiki/.mr +mr + +// ms : http://en.wikipedia.org/wiki/.ms +ms + +// mt : https://www.nic.org.mt/dotmt/ +*.mt + +// mu : http://en.wikipedia.org/wiki/.mu +// list of 2nd level tlds ? +mu + +// museum : http://about.museum/naming/ +// there are 2nd-level TLD's, but there's no list +museum + +// mv : http://en.wikipedia.org/wiki/.mv +*.mv + +// mw : http://www.registrar.mw/ +mw +ac.mw +biz.mw +co.mw +com.mw +coop.mw +edu.mw +gov.mw +int.mw +net.mw +org.mw + +// mx : http://www.nic.mx/ +*.mx + +// my : http://www.mynic.net.my/ +*.my + +// mz : http://www.gobin.info/domainname/mz-template.doc +*.mz + +// na : http://www.na-nic.com.na/ +// list of 2nd level tlds ? +na + +// name : has 2nd-level tlds, but there's no list of them +name + +// nc : http://www.cctld.nc/ +nc + +// ne : http://en.wikipedia.org/wiki/.ne +ne + +// net : http://en.wikipedia.org/wiki/.net +net + +// nf : http://en.wikipedia.org/wiki/.nf +nf +com.nf +net.nf +per.nf +rec.nf +web.nf +arts.nf +firm.nf +info.nf +other.nf +store.nf + +// ng : http://psg.com/dns/ng/ +ng + +// ni : http://www.nic.ni/dominios.htm +*.ni + +// nl : http://www.domain-registry.nl/ace.php/c,728,122,,,,Home.html +nl + +// no : http://www.norid.no/regelverk/index.en.html +no +fhs.no +vgs.no +fylkesbibl.no +folkebibl.no +museum.no +idrett.no +mil.no +stat.no +dep.no +kommune.no +herad.no +priv.no +// no geographical names : http://www.norid.no/regelverk/vedlegg-b.en.html +// counties +aa.no +ah.no +bu.no +fm.no +hl.no +hm.no +jan-mayen.no +mr.no +nl.no +nt.no +of.no +ol.no +oslo.no +rl.no +sf.no +st.no +svalbard.no +tm.no +tr.no +va.no +vf.no +// primary and lower secondary schools per county +gs.aa.no +gs.ah.no +gs.bu.no +gs.fm.no +gs.hl.no +gs.hm.no +gs.jan-mayen.no +gs.mr.no +gs.nl.no +gs.nt.no +gs.of.no +gs.ol.no +gs.oslo.no +gs.rl.no +gs.sf.no +gs.st.no +gs.svalbard.no +gs.tm.no +gs.tr.no +gs.va.no +gs.vf.no +// cities +akrehamn.no +Ã¥krehamn.no +algard.no +Ã¥lgÃ¥rd.no +arna.no +brumunddal.no +bryne.no +bronnoysund.no +brønnøysund.no +drobak.no +drøbak.no +egersund.no +fetsund.no +floro.no +florø.no +fredrikstad.no +hokksund.no +honefoss.no +hønefoss.no +jessheim.no +jorpeland.no +jørpeland.no +kirkenes.no +kopervik.no +krokstadelva.no +langevag.no +langevÃ¥g.no +leirvik.no +mjondalen.no +mjøndalen.no +mo-i-rana.no +mosjoen.no +mosjøen.no +nesoddtangen.no +orkanger.no +osoyro.no +osøyro.no +raholt.no +rÃ¥holt.no +sandnessjoen.no +sandnessjøen.no +skedsmokorset.no +slattum.no +spjelkavik.no +stathelle.no +stavern.no +stjordalshalsen.no +stjørdalshalsen.no +tananger.no +tranby.no +vossevangen.no +// communities +afjord.no +Ã¥fjord.no +agdenes.no +al.no +Ã¥l.no +alesund.no +Ã¥lesund.no +alstahaug.no +alta.no +áltá.no +alaheadju.no +álaheadju.no +alvdal.no +amli.no +Ã¥mli.no +amot.no +Ã¥mot.no +andebu.no +andoy.no +andøy.no +andasuolo.no +ardal.no +Ã¥rdal.no +aremark.no +arendal.no +Ã¥s.no +aseral.no +Ã¥seral.no +asker.no +askim.no +askvoll.no +askoy.no +askøy.no +asnes.no +Ã¥snes.no +audnedaln.no +aukra.no +aure.no +aurland.no +aurskog-holand.no +aurskog-høland.no +austevoll.no +austrheim.no +averoy.no +averøy.no +balestrand.no +ballangen.no +balat.no +bálát.no +balsfjord.no +bahccavuotna.no +báhccavuotna.no +bamble.no +bardu.no +beardu.no +beiarn.no +bajddar.no +bájddar.no +baidar.no +báidár.no +berg.no +bergen.no +berlevag.no +berlevÃ¥g.no +bearalvahki.no +bearalváhki.no +bindal.no +birkenes.no +bjarkoy.no +bjarkøy.no +bjerkreim.no +bjugn.no +bodo.no +bodø.no +badaddja.no +bÃ¥dÃ¥ddjÃ¥.no +budejju.no +bokn.no +bremanger.no +bronnoy.no +brønnøy.no +bygland.no +bykle.no +barum.no +bærum.no +bo.telemark.no +bø.telemark.no +bo.nordland.no +bø.nordland.no +bievat.no +bievát.no +bomlo.no +bømlo.no +batsfjord.no +bÃ¥tsfjord.no +bahcavuotna.no +báhcavuotna.no +dovre.no +drammen.no +drangedal.no +dyroy.no +dyrøy.no +donna.no +dønna.no +eid.no +eidfjord.no +eidsberg.no +eidskog.no +eidsvoll.no +eigersund.no +elverum.no +enebakk.no +engerdal.no +etne.no +etnedal.no +evenes.no +evenassi.no +evenášši.no +evje-og-hornnes.no +farsund.no +fauske.no +fuossko.no +fuoisku.no +fedje.no +fet.no +finnoy.no +finnøy.no +fitjar.no +fjaler.no +fjell.no +flakstad.no +flatanger.no +flekkefjord.no +flesberg.no +flora.no +fla.no +flÃ¥.no +folldal.no +forsand.no +fosnes.no +fredrikstad.no +frei.no +frogn.no +froland.no +frosta.no +frana.no +fræna.no +froya.no +frøya.no +fusa.no +fyresdal.no +forde.no +førde.no +gamvik.no +gangaviika.no +gáŋgaviika utf8 ! +gaular.no +gausdal.no +gildeskal.no +gildeskÃ¥l.no +giske.no +gjemnes.no +gjerdrum.no +gjerstad.no +gjesdal.no +gjovik.no +gjøvik.no +gloppen.no +gol.no +gran.no +grane.no +granvin.no +gratangen.no +grimstad.no +grong.no +kraanghke.no +krÃ¥anghke.no +grue.no +gulen.no +hadsel.no +halden.no +halsa.no +hamar.no +hamaroy.no +habmer.no +hábmer.no +hapmir.no +hápmir.no +hammerfest.no +hammarfeasta.no +hámmárfeasta.no +haram.no +hareid.no +harstad.no +hasvik.no +aknoluokta.no +ákÅ‹oluokta.no utf8 ! +hattfjelldal.no +aarborte.no +haugesund.no +hemne.no +hemnes.no +hemsedal.no +heroy.more-og-romsdal.no +herøy.møre-og-romsdal.no +heroy.nordland.no +herøy.nordland.no +hitra.no +hjartdal.no +hjelmeland.no +hobol.no +hobøl.no +hof.no +hol.no +hole.no +holmestrand.no +holtalen.no +holtÃ¥len.no +hornindal.no +horten.no +hurdal.no +hurum.no +hvaler.no +hyllestad.no +hagebostad.no +hægebostad.no +hoyanger.no +høyanger.no +hoylandet.no +høylandet.no +ha.no +hÃ¥.no +ibestad.no +inderoy.no +inderøy.no +iveland.no +jevnaker.no +jondal.no +jolster.no +jølster.no +karasjok.no +karasjohka.no +kárášjohka.no +karlsoy.no +galsa.no +gálsá.no +karmoy.no +karmøy.no +kautokeino.no +guovdageaidnu.no +klepp.no +klabu.no +klæbu.no +kongsberg.no +kongsvinger.no +kragero.no +kragerø.no +kristiansand.no +kristiansund.no +krodsherad.no +krødsherad.no +kvalsund.no +rahkkeravju.no +ráhkkerávju.no +kvam.no +kvinesdal.no +kvinnherad.no +kviteseid.no +kvitsoy.no +kvitsøy.no +kvafjord.no +kvæfjord.no +giehtavuoatna.no +kvanangen.no +kvænangen.no +navuotna.no +návuotna.no +kafjord.no +kÃ¥fjord.no +gaivuotna.no +gáivuotna.no +larvik.no +lavangen.no +lavagis.no +loabat.no +loabát.no +lebesby.no +davvesiida.no +leikanger.no +leirfjord.no +leka.no +leksvik.no +lenvik.no +leangaviika.no +lea?gaviika.no +lesja.no +levanger.no +lier.no +lierne.no +lillehammer.no +lillesand.no +lindesnes.no +lindas.no +lindÃ¥s.no +lom.no +loppa.no +lahppi.no +láhppi.no +lund.no +lunner.no +luroy.no +lurøy.no +luster.no +lyngdal.no +lyngen.no +ivgu.no +lardal.no +Lærdal +lerdal.no +lærdal.no +lodingen.no +lødingen.no +lorenskog.no +lørenskog.no +loten.no +løten.no +malvik.no +masoy.no +mÃ¥søy.no +muosat.no +muosát.no +mandal.no +marker.no +marnardal.no +masfjorden.no +meland.no +meldal.no +melhus.no +meloy.no +meløy.no +meraker.no +merÃ¥ker.no +moareke.no +moÃ¥reke.no +midsund.no +midtre-gauldal.no +modalen.no +modum.no +molde.no +moskenes.no +moss.no +mosvik.no +malselv.no +mÃ¥lselv.no +malatvuopmi.no +málatvuopmi.no +namdalseid.no +aejrie.no +namsos.no +namsskogan.no +naamesjevuemie.no +nååmesjevuemie.no +laakesvuemie.no +nannestad.no +narvik.no +narviika.no +naustdal.no +nedre-eiker.no +nes.akershus.no +nes.buskerud.no +nesna.no +nesodden.no +nesseby.no +unjarga.no +unjárga.no +nesset.no +nissedal.no +nittedal.no +nord-aurdal.no +nord-fron.no +nord-odal.no +norddal.no +nordkapp.no +davvenjarga.no +davvenjárga.no +nordre-land.no +nordreisa.no +raisa.no +ráisa.no +nore-og-uvdal.no +notodden.no +naroy.no +nærøy.no +notteroy.no +nøtterøy.no +odda.no +oksnes.no +`øksnes.no +oppdal.no +oppegard.no +oppegÃ¥rd.no +orkdal.no +orland.no +ørland.no +orskog.no +ørskog.no +orsta.no +ørsta.no +os.hedmark.no +os.hordaland.no +osen.no +osteroy.no +osterøy.no +ostre-toten.no +østre-toten.no +overhalla.no +ovre-eiker.no +øvre-eiker.no +oyer.no +øyer.no +oygarden.no +øygarden.no +oystre-slidre.no +øystre-slidre.no +porsanger.no +porsangu.no +porsáŋgu.no utf8 ! +porsgrunn.no +radoy.no +radøy.no +rakkestad.no +rana.no +ruovat.no +randaberg.no +rauma.no +rendalen.no +rennebu.no +rennesoy.no +rennesøy.no +rindal.no +ringebu.no +ringerike.no +ringsaker.no +rissa.no +risor.no +risør.no +roan.no +rollag.no +rygge.no +ralingen.no +rælingen.no +rodoy.no +rødøy.no +romskog.no +rømskog.no +roros.no +røros.no +rost.no +røst.no +royken.no +røyken.no +royrvik.no +røyrvik.no +rade.no +rÃ¥de.no +salangen.no +siellak.no +saltdal.no +sálát.no +sálat.no +samnanger.no +sande.more-og-romsdal.no +sande.møre-og-romsdal.no +sande.vestfold.no +sandefjord.no +sandnes.no +sandoy.no +sandøy.no +sarpsborg.no +sauda.no +sauherad.no +sel.no +selbu.no +selje.no +seljord.no +sigdal.no +siljan.no +sirdal.no +skaun.no +skedsmo.no +ski.no +skien.no +skiptvet.no +skjervoy.no +skjervøy.no +skierva.no +skiervá.no +skjak.no +skjÃ¥k.no +skodje.no +skanland.no +skÃ¥nland.no +skanit.no +skánit.no +smola.no +smøla.no +snillfjord.no +snasa.no +snÃ¥sa.no +snoasa.no +snaase.no +snÃ¥ase.no +sogndal.no +sokndal.no +sola.no +solund.no +songdalen.no +sortland.no +spydeberg.no +stange.no +stavanger.no +steigen.no +steinkjer.no +stjordal.no +stjørdal.no +stokke.no +stor-elvdal.no +stord.no +stordal.no +storfjord.no +omasvuotna.no +strand.no +stranda.no +stryn.no +sula.no +suldal.no +sund.no +sunndal.no +surnadal.no +sveio.no +svelvik.no +sykkylven.no +sogne.no +søgne.no +somna.no +sømna.no +sondre-land.no +søndre-land.no +sor-aurdal.no +sør-aurdal.no +sor-fron.no +sør-fron.no +sor-odal.no +sør-odal.no +sor-varanger.no +sør-varanger.no +matta-varjjat.no +mátta-várjjat.no +sorfold.no +sørfold.no +sorreisa.no +sørreisa.no +sorum.no +sørum.no +tana.no +deatnu.no +time.no +tingvoll.no +tinn.no +tjeldsund.no +dielddanuorri.no +tjome.no +tjøme.no +tokke.no +tolga.no +torsken.no +tranoy.no +tranøy.no +tromso.no +tromsø.no +tromsa.no +romsa.no +trondheim.no +troandin.no +trysil.no +trana.no +træna.no +trogstad.no +trøgstad.no +tvedestrand.no +tydal.no +tynset.no +tysfjord.no +divtasvuodna.no +divttasvuotna.no +tysnes.no +tysvar.no +tysvær.no +tonsberg.no +tønsberg.no +ullensaker.no +ullensvang.no +ulvik.no +utsira.no +vadso.no +vadsø.no +cahcesuolo.no +cáhcesuolo.no +vaksdal.no +valle.no +vang.no +vanylven.no +vardo.no +vardø.no +varggat.no +várggát.no +vefsn.no +vaapste.no +vega.no +vegarshei.no +vegÃ¥rshei.no +vennesla.no +verdal.no +verran.no +vestby.no +vestnes.no +vestre-slidre.no +vestre-toten.no +vestvagoy.no +vestvÃ¥gøy.no +vevelstad.no +vik.no +vikna.no +vindafjord.no +volda.no +voss.no +varoy.no +værøy.no +vagan.no +vÃ¥gan.no +voagat.no +vagsoy.no +vÃ¥gsøy.no +vaga.no +vÃ¥gÃ¥.no +valer.ostfold.no +vÃ¥ler.østfold.no +valer.hedmark.no +vÃ¥ler.hedmark.no + +// np : http://www.mos.com.np/register.html +*.np + +// nr : http://cenpac.net.nr/dns/index.html +nr +biz.nr +info.nr +gov.nr +edu.nr +org.nr +net.nr +com.nr + +// nu : http://en.wikipedia.org/wiki/.nu +nu + +// nz : http://en.wikipedia.org/wiki/.nz +*.nz + +// om : http://en.wikipedia.org/wiki/.om +*.om + +// org : http://en.wikipedia.org/wiki/.og +org + +// pa : http://www.nic.pa/ +*.pa + +// pe : http://www.nic.pe/normas-proced-i.htm +*.pe + +// pf : http://www.gobin.info/domainname/formulaire-pf.pdf +pf +com.pf +org.pf +edu.pf + +// pg : http://en.wikipedia.org/wiki/.pg +*.pg + +// ph : http://www.domains.ph/FAQ2.asp +// list of 2nd level tlds ? +ph +com.ph +net.ph +org.ph +gov.ph +edu.ph +ngo.ph +mil.ph + +// pk : http://pk5.pknic.net.pk/pk5/msgNamepk.PK +pk +com.pk +net.pk +edu.pk +org.pk +fam.pk +biz.pk +web.pk +gov.pk +gob.pk +gok.pk +gon.pk +gop.pk +gos.pk +goa.pk +info.pk + +// pl : http://www.dns.pl/english/ +pl +// NASK functional domains (nask.pl / dns.pl) : http://www.dns.pl/english/dns-funk.html +aid.pl +agro.pl +atm.pl +auto.pl +biz.pl +com.pl +edu.pl +gmina.pl +gsm.pl +info.pl +mail.pl +miasta.pl +media.pl +mil.pl +net.pl +nieruchomosci.pl +nom.pl +org.pl +pc.pl +powiat.pl +priv.pl +realestate.pl +rel.pl +sex.pl +shop.pl +sklep.pl +sos.pl +szkola.pl +targi.pl +tm.pl +tourism.pl +travel.pl +turystyka.pl +// ICM functional domains (icm.edu.pl) +6bone.pl +art.pl +mbone.pl +// Government domains (administred by ippt.gov.pl) +gov.pl +uw.gov.pl +um.gov.pl +ug.gov.pl +upow.gov.pl +starostwo.gov.pl +so.gov.pl +sr.gov.pl +po.gov.pl +pa.gov.pl +// other functional domains +med.pl +ngo.pl +irc.pl +usenet.pl +// NASK geographical domains : http://www.dns.pl/english/dns-regiony.html +augustow.pl +babia-gora.pl +bedzin.pl +beskidy.pl +bialowieza.pl +bialystok.pl +bielawa.pl +bieszczady.pl +boleslawiec.pl +bydgoszcz.pl +bytom.pl +cieszyn.pl +czeladz.pl +czest.pl +dlugoleka.pl +elblag.pl +elk.pl +glogow.pl +gniezno.pl +gorlice.pl +grajewo.pl +ilawa.pl +jaworzno.pl +jelenia-gora.pl +jgora.pl +kalisz.pl +kazimierz-dolny.pl +karpacz.pl +kartuzy.pl +kaszuby.pl +katowice.pl +kepno.pl +ketrzyn.pl +klodzko.pl +kobierzyce.pl +kolobrzeg.pl +konin.pl +konskowola.pl +kutno.pl +lapy.pl +lebork.pl +legnica.pl +lezajsk.pl +limanowa.pl +lomza.pl +lowicz.pl +lubin.pl +lukow.pl +malbork.pl +malopolska.pl +mazowsze.pl +mazury.pl +mielec.pl +mielno.pl +mragowo.pl +naklo.pl +nowaruda.pl +nysa.pl +olawa.pl +olecko.pl +olkusz.pl +olsztyn.pl +opoczno.pl +opole.pl +ostroda.pl +ostroleka.pl +ostrowiec.pl +ostrowwlkp.pl +pila.pl +pisz.pl +podhale.pl +podlasie.pl +polkowice.pl +pomorze.pl +pomorskie.pl +prochowice.pl +pruszkow.pl +przeworsk.pl +pulawy.pl +radom.pl +rawa-maz.pl +rybnik.pl +rzeszow.pl +sanok.pl +sejny.pl +slask.pl +slupsk.pl +sosnowiec.pl +stalowa-wola.pl +skoczow.pl +starachowice.pl +stargard.pl +suwalki.pl +swidnica.pl +swiebodzin.pl +swinoujscie.pl +szczecin.pl +szczytno.pl +tarnobrzeg.pl +tgory.pl +turek.pl +tychy.pl +ustka.pl +walbrzych.pl +warmia.pl +warszawa.pl +waw.pl +wegrow.pl +wielun.pl +wlocl.pl +wloclawek.pl +wodzislaw.pl +wolomin.pl +wroclaw.pl +zachpomor.pl +zagan.pl +zarow.pl +zgora.pl +zgorzelec.pl +// TASK geographical domains (www.task.gda.pl/uslugi/dns) +gda.pl +gdansk.pl +gdynia.pl +sopot.pl +// other geographical domains +gliwice.pl +krakow.pl +poznan.pl +wroc.pl +zakopane.pl + +// pn : http://www.government.pn/PnRegistry/policies.htm +pn +gov.pn +co.pn +org.pn +edu.pn +net.pn + +// pr : http://www.nic.pr/index.asp?f=1 +pr +com.pr +net.pr +org.pr +gov.pr +edu.pr +isla.pr +pro.pr +biz.pr +info.pr +name.pr +// these aren't mentioned on nic.pr, but on http://en.wikipedia.org/wiki/.pr +est.pr +prof.pr +ac.pr + +// pro : http://www.nic.pro/support_faq.htm +pro +aca.pro +bar.pro +cpa.pro +jur.pro +law.pro +med.pro +eng.pro + +// ps : http://en.wikipedia.org/wiki/.ps +// list of 2nd level tlds ? +ps +edu.ps +gov.ps +sec.ps +plo.ps +com.ps +org.ps +net.ps + +// pt : http://online.dns.pt/dns/start_dns +pt +net.pt +gov.pt +org.pt +edu.pt +int.pt +publ.pt +com.pt +nome.pt + +// pw : http://en.wikipedia.org/wiki/.pw +*.pw + +// py : http://www.nic.py/faq_a.html#faq_b +*.py + +// qa : http://www.qatar.net.qa/services/virtual.htm +*.qa + +// re : http://www.afnic.re/obtenir/chartes/nommage-re/annexe-descriptifs +re +com.re +asso.re +nom.re + +// ro : http://www.rotld.ro/ +ro +com.ro +org.ro +tm.ro +nt.ro +nom.ro +info.ro +rec.ro +arts.ro +firm.ro +store.ro +www.ro + +// ru : http://en.wikipedia.org/wiki/.ru +ru +com.ru +net.ru +org.ru +pp.ru +int.ru +// there should be geo-names like msk.ru, but I didn't find a list + +// rw : http://www.nic.rw/cgi-bin/policy.pl +rw +gov.rw +net.rw +edu.rw +ac.rw +com.rw +co.rw +int.rw +mil.rw +gouv.rw + +// sa : http://www.saudinic.net.sa/page.php?page=1&lang=1 +*.sa + +// sb : http://www.sbnic.net.sb/ +*.sb + +// sc : http://www.nic.sc/ +sc +com.sc +gov.sc +net.sc +org.sc +edu.sc + +// sd : http://www.isoc.sd/sudanic.isoc.sd/billing_pricing.htm +sd +com.sd +net.sd +org.sd +edu.sd +med.sd +tv.sd +gov.sd +info.sd + +// se : http://en.wikipedia.org/wiki/.se +se +org.se +pp.se +tm.se +parti.se +press.se +mil.se +// se geographical names +ab.se +c.se +d.se +e.se +f.se +g.se +h.se +i.se +k.se +m.se +n.se +o.se +s.se +t.se +u.se +w.se +x.se +y.se +z.se +ac.se +bd.se + +// sg : http://www.nic.net.sg/sub_policies_agreement/2ld.html +sg +com.sg +net.sg +org.sg +gov.sg +edu.sg +per.sg + +// sh : http://www.nic.sh/rules.html +// list of 2nd level domains ? +sh + +// si : http://en.wikipedia.org/wiki/.si +si + +// sk : http://en.wikipedia.org/wiki/.sk +sk + +// sl : http://en.wikipedia.org/wiki/.sl +// list of 2nd level domains ? +sl + +// sm : http://en.wikipedia.org/wiki/.sm +sm + +// sn : http://en.wikipedia.org/wiki/.sn +// list of 2nd level domains ? +sn + +// sr : http://en.wikipedia.org/wiki/.sr +sr + +// st : http://www.nic.st/html/policyrules/ +st + +// su : http://en.wikipedia.org/wiki/.su +su + +// sv : http://www.svnet.org.sv/svpolicy.html +*.sv + +// sy : http://www.gobin.info/domainname/sy.doc +*.sy + +// sz : http://en.wikipedia.org/wiki/.sz +// list of 2nd level domains ? +sz + +// tc : http://en.wikipedia.org/wiki/.tc +tc + +// td : http://en.wikipedia.org/wiki/.td +td + +// tf : http://en.wikipedia.org/wiki/.tf +tf + +// tg : http://en.wikipedia.org/wiki/.tg +// list of 2nd level domains ? +tg + +// th : http://en.wikipedia.org/wiki/.th +*.th + +// tj : http://www.nic.tj/policy.htm +tj +ac.tj +biz.tj +com.tj +co.tj +edu.tj +int.tj +name.tj +net.tj +org.tj +web.tj +gov.tj +go.tj +mil.tj + +// tk : http://en.wikipedia.org/wiki/.tk +tk + +// tl : http://en.wikipedia.org/wiki/.tl +// list of 2nd level tlds ? +tl + +// tm : http://www.nic.tm/rules.html +// list of 2nd level tlds ? +tm + +// tn : http://en.wikipedia.org/wiki/.tn +// list of 2nd level tlds ? +tn + +// to : http://en.wikipedia.org/wiki/.to +// list of 2nd level tlds ? +to + +// tr : http://en.wikipedia.org/wiki/.tr +*.tr + +// travel : http://en.wikipedia.org/wiki/.travel +travel + +// tt : http://www.nic.tt/ +tt +co.tt +com.tt +org.tt +net.tt +biz.tt +info.tt +pro.tt +int.tt +coop.tt +jobs.tt +mobi.tt +travel.tt +museum.tt +aero.tt +name.tt +gov.tt +edu.tt + +// tv : http://en.wikipedia.org/wiki/.tv +// list of 2nd level tlds ? +tv + +// tw : http://en.wikipedia.org/wiki/.tw +tw +edu.tw +gov.tw +mil.tw +com.tw +net.tw +org.tw +idv.tw +game.tw +ebiz.tw +club.tw +網路.tw utf8 ! +組織.tw utf8 ! +商業.tw utf8 ! + +// tz : http://en.wikipedia.org/wiki/.tz +*.tz + +// ua : http://www.nic.net.ua/ +ua +com.ua +edu.ua +gov.ua +net.ua +org.ua +// ua geo-names +cherkassy.ua +chernigov.ua +chernovtsy.ua +ck.ua +cn.ua +crimea.ua +cv.ua +dn.ua +dnepropetrovsk.ua +donetsk.ua +dp.ua +if.ua +ivano-frankivsk.ua +kh.ua +kharkov.ua +kherson.ua +kiev.ua +kirovograd.ua +km.ua +kr.ua +ks.ua +lg.ua +lugansk.ua +lutsk.ua +lviv.ua +mk.ua +nikolaev.ua +od.ua +odessa.ua +pl.ua +poltava.ua +rovno.ua +rv.ua +sebastopol.ua +sumy.ua +te.ua +ternopil.ua +vinnica.ua +vn.ua +zaporizhzhe.ua +zp.ua +uz.ua +uzhgorod.ua +zhitomir.ua +zt.ua + +// ug : http://www.registry.co.ug/ +ug +co.ug +ac.ug +sc.ug +go.ug +ne.ug +or.ug + +// uk : http://en.wikipedia.org/wiki/.uk +*.uk +*.sch.uk +!bl.uk +!british-library.uk +!icnet.uk +!jet.uk +!nel.uk +!nls.uk +!national-library-scotland.uk +!parliament.uk + +// us : http://en.wikipedia.org/wiki/.us +us +dni.us +fed.us +isa.us +kids.us +nsn.us +// us geographic names +ak.us +al.us +ar.us +az.us +ca.us +co.us +ct.us +dc.us +de.us +fl.us +ga.us +hi.us +ia.us +id.us +il.us +in.us +ks.us +ky.us +la.us +ma.us +md.us +me.us +mi.us +mn.us +mo.us +ms.us +mt.us +nc.us +nd.us +ne.us +nh.us +nj.us +nm.us +nv.us +ny.us +oh.us +ok.us +or.us +pa.us +ri.us +sc.us +sd.us +tn.us +tx.us +ut.us +vt.us +va.us +wa.us +wi.us +wv.us +wy.us +// the following rules would be only valid under the geo-name, but we can't express that +// *.*.us cities, counties, parishes, and townships (locality.state.us) +// !ci.*.*.us city government agencies (subdomain under locality) +// !town.*.*.us town government agencies (subdomain under locality) +// !co.*.*.us county government agencies (subdomain under locality) +// k12.*.us public school districts +// pvt.k12.*.us private schools +// cc.*.us community colleges +// tec.*.us technical and vocational schools +// lib.*.us state, regional, city, and county libraries +// state.*.us state government agencies +// gen.*.us general independent entities (groups not fitting into the above categories) + +// uy : http://www.antel.com.uy/ +*.uy + +// uz : http://www.reg.uz/registerr.html +// are there other 2nd level tlds ? +uz +com.uz +co.uz + +// va : http://en.wikipedia.org/wiki/.va +va + +// vc : http://en.wikipedia.org/wiki/.vc +// list of 2nd level tlds ? +vc + +// ve : http://registro.nic.ve/nicve/registro/index.html +*.ve + +// vg : http://en.wikipedia.org/wiki/.vg +vg + +// vi : http://www.nic.vi/Domain_Rules/body_domain_rules.html +vi +com.vi +org.vi +edu.vi +gov.vi + +// vn : https://www.dot.vn/vnnic/vnnic/domainregistration.jsp +vn +com.vn +net.vn +org.vn +edu.vn +gov.vn +int.vn +ac.vn +biz.vn +info.vn +name.vn +pro.vn +health.vn + +// vu : http://en.wikipedia.org/wiki/.vu +// list of 2nd level tlds ? +vu + +// ws : http://en.wikipedia.org/wiki/.ws +ws + +// ye : http://www.y.net.ye/services/domain_name.htm +*.ye + +// yu : http://www.nic.yu/pravilnik-e.html +*.yu + +// za : http://www.zadna.org.za/slds.html +*.za + +// zm : http://en.wikipedia.org/wiki/.zm +*.zm + +// zw : http://en.wikipedia.org/wiki/.zw +*.zw + diff --git a/commons/src/main/resources/org/archive/i18n/LocaleCache_en.utf8 b/commons/src/main/resources/org/archive/i18n/LocaleCache_en.utf8 new file mode 100644 index 00000000..7ec70d91 --- /dev/null +++ b/commons/src/main/resources/org/archive/i18n/LocaleCache_en.utf8 @@ -0,0 +1,9 @@ +# LocaleCache.en.utf8 -- data for LocaleCache in English + +hello: +Hello, world! + +multiline: +Most modern calendars mar the sweet simplicity of our lives by reminding us +that each day that passes is the anniversary of some perfectly uninteresting +event. diff --git a/commons/src/main/resources/org/archive/i18n/LocaleCache_en_CA.utf8 b/commons/src/main/resources/org/archive/i18n/LocaleCache_en_CA.utf8 new file mode 100644 index 00000000..d28b4cce --- /dev/null +++ b/commons/src/main/resources/org/archive/i18n/LocaleCache_en_CA.utf8 @@ -0,0 +1,9 @@ +# LocaleCache.en_US.utf8 -- test data for LocaleCache in faux Canadian english + +hello: +Hello world, eh? + +multiline: +Most modern calendars mar the sweet simplicity of our lives by reminding us +that each day that passes is the anniversary of some perfectly uninteresting +event, eh? diff --git a/commons/src/main/resources/org/archive/i18n/LocaleCache_en_US_borkborkbork.utf8 b/commons/src/main/resources/org/archive/i18n/LocaleCache_en_US_borkborkbork.utf8 new file mode 100644 index 00000000..1b81e6d4 --- /dev/null +++ b/commons/src/main/resources/org/archive/i18n/LocaleCache_en_US_borkborkbork.utf8 @@ -0,0 +1,10 @@ +# LocaleCache.en_US_borkborkbork.utf8 -- data for LocaleCache in Swedish Chef + +hello: +Yorn desh born, der ritt de gitt der gue, orn desh, dee born desh, +de umn, børk børk børk! + +multiline: +Must mudern celenders mer zee sveet seemplicity ooff oooor leefes by remeending +us thet iech dey thet pesses is zee unneefersery ooff sume-a perffectly +uneenteresting ifent, børk børk børk! diff --git a/commons/src/main/resources/org/archive/i18n/LocaleCache_jp.utf8 b/commons/src/main/resources/org/archive/i18n/LocaleCache_jp.utf8 new file mode 100644 index 00000000..3093ee58 --- /dev/null +++ b/commons/src/main/resources/org/archive/i18n/LocaleCache_jp.utf8 @@ -0,0 +1,9 @@ +# LocaleCache_jp.utf8 -- test data for LocaleCache in Japanese +# (Translation provided by google; it's probably not accurate.) + +hello: +ã“ã‚“ã«ã¡ã¯ã€ä¸–ç•Œ! + +multiline: +ã»ã¨ã‚“ã©ã®ç¾ä»£ã‚«ãƒ¬ãƒ³ãƒ€ãƒ¼ã¯ç§é”ã«ãれ毎日æ€ã„出ã•ã›ã‚‹ã“ã¨ã«ã‚ˆã£ã¦ç§é”ã®ç”Ÿå‘½ã®ç”˜ã„簡易性を傷ã¤ã‘るパス㌠+完全ã«é€€å±ˆãªã§ã事ã®è¨˜å¿µæ—¥ã§ã‚ã‚‹ã“ã¨ã€‚ \ No newline at end of file diff --git a/commons/src/main/resources/org/archive/util/version.txt b/commons/src/main/resources/org/archive/util/version.txt new file mode 100644 index 00000000..96a22a1b --- /dev/null +++ b/commons/src/main/resources/org/archive/util/version.txt @@ -0,0 +1 @@ +${pom.version} diff --git a/commons/src/site/overview.html b/commons/src/site/overview.html new file mode 100644 index 00000000..13513962 --- /dev/null +++ b/commons/src/site/overview.html @@ -0,0 +1,3 @@ + +General Java utility libraries. + diff --git a/commons/src/test/java/org/archive/io/BufferedSeekInputStreamTest.java b/commons/src/test/java/org/archive/io/BufferedSeekInputStreamTest.java new file mode 100644 index 00000000..1b16332c --- /dev/null +++ b/commons/src/test/java/org/archive/io/BufferedSeekInputStreamTest.java @@ -0,0 +1,71 @@ +/* BufferedSeekInputStreamTest +* +* Created on September 18, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.io; + +import java.util.Random; + +import junit.framework.TestCase; + + +/** + * Unit test for BufferedSeekInputStream. The tests do some random + * repositioning in the stream to make sure the buffer is always valid. + * + * @author pjack + */ +public class BufferedSeekInputStreamTest extends TestCase { + + + private static byte[] TEST_DATA = makeTestData(); + + public void testPosition() throws Exception { + Random random = new Random(); + ArraySeekInputStream asis = new ArraySeekInputStream(TEST_DATA); + BufferedSeekInputStream bsis = new BufferedSeekInputStream(asis, 11); + for (int i = 0; i < TEST_DATA.length; i++) { + byte b = (byte)bsis.read(); + assertEquals(TEST_DATA[i], b); + } + for (int i = 0; i < 1000; i++) { + int index = random.nextInt(TEST_DATA.length); + bsis.position(index); + char expected = (char)((int)TEST_DATA[index] & 0xFF); + char read = (char)(bsis.read() & 0xFF); + assertEquals(expected, read); + } + } + + + private static byte[] makeTestData() { + String s = "If the dull substance of my flesh were thought\n" + + "Injurious distance could not stop my way\n" + + "For then, despite of space, I would be brought\n" + + "From limits far remote where thou dost stay.\n"; + byte[] r = new byte[s.length()]; + for (int i = 0; i < r.length; i++) { + r[i] = (byte)s.charAt(i); +// r[i] = (byte)s.charAt(i); + } + return r; + } +} diff --git a/commons/src/test/java/org/archive/io/GzippedInputStreamTest.java b/commons/src/test/java/org/archive/io/GzippedInputStreamTest.java new file mode 100644 index 00000000..32187934 --- /dev/null +++ b/commons/src/test/java/org/archive/io/GzippedInputStreamTest.java @@ -0,0 +1,167 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io; + +import it.unimi.dsi.fastutil.io.RepositionableStream; + +import java.io.BufferedOutputStream; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Iterator; + +import org.archive.util.TmpDirTestCase; + +/** + * @author stack + * @version $Date$, $Revision$ + */ +public class GzippedInputStreamTest extends TmpDirTestCase { + /** + * Number of records in gzip member file. + */ + final static int GZIPMEMBER_COUNT = 4; + final static String TEXT = "Some old text to compress."; + // Create file to use in tests below. + private File compressedFile = null; + + protected void setUp() throws Exception { + super.setUp(); + this.compressedFile = createMultiGzipMembers(); + } + + protected void tearDown() throws Exception { + if (this.compressedFile != null) { + this.compressedFile.delete(); + } + super.tearDown(); + } + + public static void main(String [] args) { + junit.textui.TestRunner.run(GzippedInputStreamTest.class); + } + + protected class RepositionableRandomAccessInputStream + extends RandomAccessInputStream + implements RepositionableStream { + public RepositionableRandomAccessInputStream(final File file) + throws IOException { + super(file); + } + + public RepositionableRandomAccessInputStream(final File file, + final long offset) + throws IOException { + super(file, offset); + } + } + + protected File createMultiGzipMembers() throws IOException { + final File f = + new File(getTmpDir(), this.getClass().getName() + ".gz"); + OutputStream os = new BufferedOutputStream(new FileOutputStream(f)); + for (int i = 0; i < GZIPMEMBER_COUNT; i++) { + os.write(GzippedInputStream.gzip(TEXT.getBytes())); + } + os.close(); + return f; + } + + public void testCountOfMembers() + throws IOException { + InputStream is = + new RepositionableRandomAccessInputStream(this.compressedFile); + GzippedInputStream gis = new GzippedInputStream(is); + int records = 0; + // Get offset of second record. Will use it later in tests below. + long offsetOfSecondRecord = -1; + for (Iterator i = gis.iterator(); i.hasNext();) { + long offset = gis.position(); + if (records == 1) { + offsetOfSecondRecord = offset; + } + is = (InputStream)i.next(); + records++; + } + assertTrue("Record count is off " + records, + records == GZIPMEMBER_COUNT); + gis.close(); + + // Test random record read. + is = new RepositionableRandomAccessInputStream(this.compressedFile); + gis = new GzippedInputStream(is); + byte [] buffer = new byte[TEXT.length()]; + // Seek to second record, read in gzip header. + gis.gzipMemberSeek(offsetOfSecondRecord); + gis.read(buffer); + String readString = new String(buffer); + assertEquals("Failed read", TEXT, readString); + gis.close(); + + // Test the count we get makes sense after iterating through + // starting at second record. + is = new RepositionableRandomAccessInputStream(this.compressedFile, + offsetOfSecondRecord); + gis = new GzippedInputStream(is); + records = 0; + for (final Iterator i = gis.iterator(); i.hasNext(); i.next()) { + records++; + } + assertEquals(records, + GZIPMEMBER_COUNT - 1 /*We started at 2nd record*/); + gis.close(); + } + + public void testCompressedStream() throws IOException { + byte [] bytes = "test".getBytes(); + ByteArrayInputStream baos = new ByteArrayInputStream(bytes); + assertFalse(GzippedInputStream.isCompressedStream(baos)); + + byte [] gzippedMetaData = GzippedInputStream.gzip(bytes); + baos = new ByteArrayInputStream(gzippedMetaData); + assertTrue(GzippedInputStream.isCompressedStream(baos)); + + gzippedMetaData = GzippedInputStream.gzip(bytes); + final RepositionableByteArrayInputStream rbaos = + new RepositionableByteArrayInputStream(gzippedMetaData); + final int totalBytes = gzippedMetaData.length; + assertTrue(GzippedInputStream.isCompressedRepositionableStream(rbaos)); + long available = rbaos.available(); + assertEquals(available, totalBytes); + assertEquals(rbaos.position(), 0); + } + + private class RepositionableByteArrayInputStream + extends ByteArrayInputStream implements RepositionableStream { + public RepositionableByteArrayInputStream(final byte [] bytes) { + super(bytes); + } + + public void position(long p) { + this.pos = (int)p; + } + public long position() { + return this.pos; + } + } +} \ No newline at end of file diff --git a/commons/src/test/java/org/archive/io/HeaderedArchiveRecordTest.java b/commons/src/test/java/org/archive/io/HeaderedArchiveRecordTest.java new file mode 100644 index 00000000..94a216ca --- /dev/null +++ b/commons/src/test/java/org/archive/io/HeaderedArchiveRecordTest.java @@ -0,0 +1,204 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import junit.framework.TestCase; + +import org.apache.commons.httpclient.Header; +import org.archive.io.arc.ARCRecord; +import org.archive.io.warc.WARCRecord; + +public class HeaderedArchiveRecordTest extends TestCase { + private static final String HTTPHEADER = "HTTP/1.1 200 OK\r\n" + + "Last-Modified: Sun, 28 Aug 2005 14:10:55 GMT\r\n" + + "Content-Length: 108\r\n" + "Connection: close\r\n" + + "Content-Type: text/html\r\n" + "\r\n"; + private static final String BODY = "\r\n" + " \r\n" + + " Neue Seite 1\r\n" + " \r\n" + + " \r\n" + " \r\n" + ""; + + public void testParseHttpHeadersInWARC() throws IOException { + final String url = "http://foo.maths.uq.edu.au/index.html"; + // final String warcHeader = "WARC/0.10 000000000486 response " + + // url + " 20070315152520 " + + // "urn:uuid:d8b342a8-dba4-4d7f-a551-1d8184f2ff58 " + + // "application/http; msgtype=response\r\n" + + // "Checksum: sha1:IT6YEX5WHKK57GOEHV2YHTTXEP5KPM6A\r\n" + + // "IP-Address: 80.150.6.184\r\n" + + // "\r\n"; + + final String warcHeader = "WARC/0.12\r\n" + + "MIME-Version: 1.0\r\n" + + "WARC-Record-Type: response\r\n" + + "WARC-Target-URI: http://foo.maths.uq.edu.au/index.html\r\n" + + "WARC-Date: 2006-09-19T17:20:24Z\r\n" + + "WARC-Digest: sha1:IT6YEX5WHKK57GOEHV2YHTTXEP5KPM6A\r\n" + + "WARC-IP-Address: 80.150.6.184\r\n" + + "Content-ID: \r\n" + + "Content-Type: application/http; msgtype=response\r\n" + + "Content-Length: " + (HTTPHEADER.length() + BODY.length()) + "\r\n" + + "\r\n"; + + final String hdr = warcHeader + HTTPHEADER + BODY; + + WARCRecord r = new WARCRecord(new ByteArrayInputStream(hdr.getBytes()), + "READER_IDENTIFIER", 0, false, true); + HeaderedArchiveRecord har = new HeaderedArchiveRecord(r, true); + + har.skipHttpHeader(); + + byte[] b = new byte[BODY.length()]; + har.read(b); + String bodyRead = new String(b); + assertEquals(BODY, bodyRead); + assertHeaderCorrectlyParsed(har.getContentHeaders()); + assertEquals("failed to retrieve Url from metadata", har.getHeader() + .getUrl(), url); + } + + public void testParseHttpHeadersInARC() throws IOException { + final int len = HTTPHEADER.length() + BODY.length(); + final String url = "http://www.ly.gov.tw:80/accpart.htm"; + final String hdr = HTTPHEADER + BODY; + // Interesting difference between ARCRecord and WARCRecord is that the + // stream passed the ARCRecord is supposed to be just past the + // ARCRecord metadata line where as stream passed WARCRecord is at + // record start. TODO: Add to ARCRecord constructor that doesn't + // take an ArchiveRecordHeader but rather parses it from the stream. + ArchiveRecordHeader arh = new ArchiveRecordHeader() { + public int getContentBegin() { + // TODO: In ARCs, this is where http headers end and + // the content begins. Need to reconcile for generic + // HeaderedArchiveRecord processing. In this context, it + // makes sense setting it to zero -- HeaderedArchiveRecord + // will then figure it out. + return 0; + } + + public String getDate() { + return null; + } + + public String getDigest() { + return null; + } + + public Set getHeaderFieldKeys() { + return null; + } + + public Map getHeaderFields() { + return null; + } + + public Object getHeaderValue(String key) { + return null; + } + + public long getLength() { + return len; + } + + public String getMimetype() { + return null; + } + + public long getOffset() { + return 0; + } + + public String getReaderIdentifier() { + return null; + } + + public String getRecordIdentifier() { + return null; + } + + public String getUrl() { + return url; + } + + public String getVersion() { + return null; + } + + }; + ARCRecord r = new ARCRecord(new ByteArrayInputStream(hdr.getBytes()), + arh, 0, false, true, false); + + HeaderedArchiveRecord har = new HeaderedArchiveRecord(r, true); + har.skipHttpHeader(); + byte[] b = new byte[BODY.length()]; + har.read(b); + String bodyRead = new String(b); + assertEquals(BODY, bodyRead); + assertHeaderCorrectlyParsed(har.getContentHeaders()); + } + + public void testEasierParseHttpHeadersInARC() throws IOException { + final String url = "http://www.archive.org/index.htm"; + final String arcHeader = url + + " 192.168.0.1 20070515111004 text/html 167568\n"; + final String hdr = arcHeader + HTTPHEADER + BODY; + + ARCRecord r = new ARCRecord(new ByteArrayInputStream(hdr.getBytes()), + "READER_IDENTIFIER", 0, false, true, false); + + HeaderedArchiveRecord har = new HeaderedArchiveRecord(r, true); + har.skipHttpHeader(); + byte[] b = new byte[BODY.length()]; + har.read(b); + String bodyRead = new String(b); + assertEquals(BODY, bodyRead); + assertHeaderCorrectlyParsed(har.getContentHeaders()); + assertEquals("failed to retrieve Url from metadata", har.getHeader() + .getUrl(), url); + } + + private void assertHeaderCorrectlyParsed(Header[] headers) { + final List orgHeaders = Arrays.asList(HTTPHEADER.split("\r\n")); + assertEquals("not all HTTP header entries have been retrieved", + orgHeaders.size(), headers.length + 1); + + for (Header header : headers) { + assertTrue(orgHeaders.contains(header.getName() + ": " + + header.getValue())); + } + } + + public void testNoheaderWARC() throws IOException { + String b = "hello world"; + String c = "WARC/0.12\r\nContent-Type: text/plain\r\n" + + "Content-Length: " + b.length() + "\r\n\r\n" + b; + org.archive.io.warc.WARCRecord r = new org.archive.io.warc.WARCRecord( + new ByteArrayInputStream(c.getBytes()), "READER_IDENTIFIER", 0, + false, true); + HeaderedArchiveRecord har = new HeaderedArchiveRecord(r, true); + assertTrue(har.isStrict()); + } +} diff --git a/commons/src/test/java/org/archive/io/RecordingInputStreamTest.java b/commons/src/test/java/org/archive/io/RecordingInputStreamTest.java new file mode 100644 index 00000000..c5529847 --- /dev/null +++ b/commons/src/test/java/org/archive/io/RecordingInputStreamTest.java @@ -0,0 +1,138 @@ +/* RecordingInputStreamTest + * + * $Id$ + * + * Created on Aug 1, 2005 + * + * Copyright (C) 2005 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.io; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; + +import org.archive.util.TmpDirTestCase; + + +/** + * Test cases for RecordingInputStream. + * + * @author gojomo + */ +public class RecordingInputStreamTest extends TmpDirTestCase +{ + + + /* + * @see TmpDirTestCase#setUp() + */ + protected void setUp() throws Exception + { + super.setUp(); + } + + /** + * Test readFullyOrUntil soft (no exception) and hard (exception) + * length cutoffs, timeout, and rate-throttling. + * + * @throws IOException + * @throws InterruptedException + * @throws RecorderTimeoutException + */ + public void testReadFullyOrUntil() throws RecorderTimeoutException, IOException, InterruptedException + { + RecordingInputStream ris = new RecordingInputStream(16384, (new File( + getTmpDir(), "testReadFullyOrUntil").getAbsolutePath())); + ByteArrayInputStream bais = new ByteArrayInputStream( + "abcdefghijklmnopqrstuvwxyz".getBytes()); + // test soft max + ris.open(bais); + ris.setLimits(10,0,0); + ris.readFullyOrUntil(7); + ris.close(); + ReplayInputStream res = ris.getReplayInputStream(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + res.readFullyTo(baos); + assertEquals("soft max cutoff","abcdefg",new String(baos.toByteArray())); + // test hard max + bais.reset(); + baos.reset(); + ris.open(bais); + boolean exceptionThrown = false; + try { + ris.setLimits(10,0,0); + ris.readFullyOrUntil(13); + } catch (RecorderLengthExceededException ex) { + exceptionThrown = true; + } + assertTrue("hard max exception",exceptionThrown); + ris.close(); + res = ris.getReplayInputStream(); + res.readFullyTo(baos); + assertEquals("hard max cutoff","abcdefghijk", + new String(baos.toByteArray())); + // test timeout + PipedInputStream pin = new PipedInputStream(); + PipedOutputStream pout = new PipedOutputStream(pin); + ris.open(pin); + exceptionThrown = false; + trickle("abcdefghijklmnopqrstuvwxyz".getBytes(),pout); + try { + ris.setLimits(0,5000,0); + ris.readFullyOrUntil(0); + } catch (RecorderTimeoutException ex) { + exceptionThrown = true; + } + assertTrue("timeout exception",exceptionThrown); + ris.close(); + // test rate limit + bais = new ByteArrayInputStream(new byte[1024*2*5]); + ris.open(bais); + long startTime = System.currentTimeMillis(); + ris.setLimits(0,0,2); + ris.readFullyOrUntil(0); + long endTime = System.currentTimeMillis(); + long duration = endTime - startTime; + assertTrue("read too fast: "+duration,duration>=5000); + ris.close(); + } + + protected void trickle(final byte[] bytes, final PipedOutputStream pout) { + new Thread() { + public void run() { + try { + for (int i = 0; i < bytes.length; i++) { + Thread.sleep(1000); + pout.write(bytes[i]); + } + pout.close(); + } catch (IOException e) { + // do nothing + } catch (Exception e) { + System.err.print(e); + } + } + }.start(); + + } +} diff --git a/commons/src/test/java/org/archive/io/RecordingOutputStreamTest.java b/commons/src/test/java/org/archive/io/RecordingOutputStreamTest.java new file mode 100644 index 00000000..615688f9 --- /dev/null +++ b/commons/src/test/java/org/archive/io/RecordingOutputStreamTest.java @@ -0,0 +1,266 @@ +/* RecordingOutputStreamTest + * + * $Id$ + * + * Created on Jan 21, 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.io; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; + +import org.archive.util.TmpDirTestCase; + + +/** + * Test casesfor RecordingOutputStream. + * + * @author stack + */ +public class RecordingOutputStreamTest extends TmpDirTestCase +{ + /** + * Size of buffer used in tests. + */ + private static final int BUFFER_SIZE = 5; + + /** + * How much to write total to testing RecordingOutputStream. + */ + private static final int WRITE_TOTAL = 10; + + + /* + * @see TmpDirTestCase#setUp() + */ + protected void setUp() throws Exception + { + super.setUp(); + } + + /** + * Test reusing instance of RecordingOutputStream. + * + * @throws IOException Failed open of backing file or opening of + * input streams verifying recording. + */ + public void testReuse() + throws IOException + { + final String BASENAME = "testReuse"; + cleanUpOldFiles(BASENAME); + RecordingOutputStream ros = new RecordingOutputStream(BUFFER_SIZE, + (new File(getTmpDir(), BASENAME + "Bkg.txt")).getAbsolutePath()); + for (int i = 0; i < 3; i++) + { + reuse(BASENAME, ros, i); + } + } + + private void reuse(String baseName, RecordingOutputStream ros, int index) + throws IOException + { + final String BASENAME = baseName + Integer.toString(index); + File f = writeIntRecordedFile(ros, BASENAME, WRITE_TOTAL); + verifyRecording(ros, f, WRITE_TOTAL); + // Do again to test that I can get a new ReplayInputStream on same + // RecordingOutputStream. + verifyRecording(ros, f, WRITE_TOTAL); + } + + /** + * Method to test for void write(int). + * + * Uses small buffer size and small write size. Test mark and reset too. + * + * @throws IOException Failed open of backing file or opening of + * input streams verifying recording. + */ + public void testWriteint() + throws IOException + { + final String BASENAME = "testWriteint"; + cleanUpOldFiles(BASENAME); + RecordingOutputStream ros = new RecordingOutputStream(BUFFER_SIZE, + (new File(getTmpDir(), BASENAME + "Backing.txt")).getAbsolutePath()); + File f = writeIntRecordedFile(ros, BASENAME, WRITE_TOTAL); + verifyRecording(ros, f, WRITE_TOTAL); + // Do again to test that I can get a new ReplayInputStream on same + // RecordingOutputStream. + verifyRecording(ros, f, WRITE_TOTAL); + } + + /** + * Method to test for void write(byte []). + * + * Uses small buffer size and small write size. + * + * @throws IOException Failed open of backing file or opening of + * input streams verifying recording. + */ + public void testWritebytearray() + throws IOException + { + final String BASENAME = "testWritebytearray"; + cleanUpOldFiles(BASENAME); + RecordingOutputStream ros = new RecordingOutputStream(BUFFER_SIZE, + (new File(getTmpDir(), BASENAME + "Backing.txt")).getAbsolutePath()); + File f = writeByteRecordedFile(ros, BASENAME, WRITE_TOTAL); + verifyRecording(ros, f, WRITE_TOTAL); + // Do again to test that I can get a new ReplayInputStream on same + // RecordingOutputStream. + verifyRecording(ros, f, WRITE_TOTAL); + } + + /** + * Test mark and reset. + * @throws IOException + */ + public void testMarkReset() throws IOException + { + final String BASENAME = "testMarkReset"; + cleanUpOldFiles(BASENAME); + RecordingOutputStream ros = new RecordingOutputStream(BUFFER_SIZE, + (new File(getTmpDir(), BASENAME + "Backing.txt")).getAbsolutePath()); + File f = writeByteRecordedFile(ros, BASENAME, WRITE_TOTAL); + verifyRecording(ros, f, WRITE_TOTAL); + ReplayInputStream ris = ros.getReplayInputStream(); + ris.mark(10 /*Arbitrary value*/); + // Read from the stream. + ris.read(); + ris.read(); + ris.read(); + // Reset it. It should be back at zero. + ris.reset(); + assertEquals("Reset to zero", ris.read(), 0); + assertEquals("Reset to zero char 1", ris.read(), 1); + assertEquals("Reset to zero char 2", ris.read(), 2); + // Mark stream. Here. Next character should be '3'. + ris.mark(10 /* Arbitrary value*/); + ris.read(); + ris.read(); + ris.reset(); + assertEquals("Reset to zero char 3", ris.read(), 3); + } + + /** + * Record a file write. + * + * Write a file w/ characters that start at null and ascend to + * filesize. Record the writing w/ passed ros + * recordingoutputstream. Return the file recorded as result of method. + * The file output stream that is recorded is named + * basename + ".txt". + * + *

This method writes a character at a time. + * + * @param ros RecordingOutputStream to record with. + * @param basename Basename of file. + * @param size How many characters to write. + * @return Recorded output stream. + */ + private File writeIntRecordedFile(RecordingOutputStream ros, + String basename, int size) + throws IOException + { + File f = new File(getTmpDir(), basename + ".txt"); + FileOutputStream fos = new FileOutputStream(f); + ros.open(fos); + for (int i = 0; i < WRITE_TOTAL; i++) + { + ros.write(i); + } + ros.close(); + fos.close(); + assertEquals("Content-Length test", size, + ros.getResponseContentLength()); + return f; + } + + /** + * Record a file byte array write. + * + * Write a file w/ characters that start at null and ascend to + * filesize. Record the writing w/ passed ros + * recordingoutputstream. Return the file recorded as result of method. + * The file output stream that is recorded is named + * basename + ".txt". + * + *

This method writes using a byte array. + * + * @param ros RecordingOutputStream to record with. + * @param basename Basename of file. + * @param size How many characters to write. + * @return Recorded output stream. + */ + private File writeByteRecordedFile(RecordingOutputStream ros, + String basename, int size) + throws IOException + { + File f = new File(getTmpDir(), basename + ".txt"); + FileOutputStream fos = new FileOutputStream(f); + ros.open(fos); + byte [] b = new byte[size]; + for (int i = 0; i < size; i++) + { + b[i] = (byte)i; + } + ros.write(b); + ros.close(); + fos.close(); + assertEquals("Content-Length test", size, + ros.getResponseContentLength()); + return f; + } + + /** + * Verify what was written is both in the file written to and in the + * recording stream. + * + * @param ros Stream to check. + * @param f File that was recorded. Stream should have its content + * exactly. + * @param size Amount of bytes written. + * + * @exception IOException Failure reading streams. + */ + private void verifyRecording(RecordingOutputStream ros, File f, + int size) throws IOException + { + assertEquals("Recorded file size.", size, f.length()); + FileInputStream fis = new FileInputStream(f); + assertNotNull("FileInputStream not null", fis); + ReplayInputStream ris = ros.getReplayInputStream(); + assertNotNull("ReplayInputStream not null", ris); + for (int i = 0; i < size; i++) + { + assertEquals("ReplayInputStream content verification", i, + ris.read()); + assertEquals("Recorded file content verification", i, + fis.read()); + } + assertEquals("ReplayInputStream at EOF", -1, ris.read()); + fis.close(); + ris.close(); + } +} diff --git a/commons/src/test/java/org/archive/io/ReplayCharSequenceTest.java b/commons/src/test/java/org/archive/io/ReplayCharSequenceTest.java new file mode 100644 index 00000000..614e05cb --- /dev/null +++ b/commons/src/test/java/org/archive/io/ReplayCharSequenceTest.java @@ -0,0 +1,281 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io; + +import java.io.IOException; +import java.util.Date; +import java.util.logging.Logger; + +import org.archive.util.FileUtils; +import org.archive.util.TmpDirTestCase; + +/** + * Test ReplayCharSequences. + * + * @author stack, gojomo + * @version $Revision$, $Date$ + */ +public class ReplayCharSequenceTest extends TmpDirTestCase +{ + /** + * Logger. + */ + private static Logger logger = + Logger.getLogger("org.archive.io.ReplayCharSequenceFactoryTest"); + + + private static final int SEQUENCE_LENGTH = 127; + private static final int MULTIPLIER = 3; + private static final int BUFFER_SIZE = SEQUENCE_LENGTH * MULTIPLIER; + private static final int INCREMENT = 1; + + /** + * Buffer of regular content. + */ + private byte [] regularBuffer = null; + + /* + * @see TestCase#setUp() + */ + protected void setUp() throws Exception + { + super.setUp(); + this.regularBuffer = + fillBufferWithRegularContent(new byte [BUFFER_SIZE]); + } + + public void testShiftjis() throws IOException { + + // Here's the bytes for the JIS encoding of the Japanese form of Nihongo + byte[] bytes_nihongo = { + (byte) 0x1B, (byte) 0x24, (byte) 0x42, (byte) 0x46, + (byte) 0x7C, (byte) 0x4B, (byte) 0x5C, (byte) 0x38, + (byte) 0x6C, (byte) 0x1B, (byte) 0x28, (byte) 0x42, + (byte) 0x1B, (byte) 0x28, (byte) 0x42 }; + final String ENCODING = "SJIS"; + // Here is nihongo converted to JVM encoding. + String nihongo = new String(bytes_nihongo, ENCODING); + + RecordingOutputStream ros = writeTestStream( + bytes_nihongo,MULTIPLIER, + "testShiftjis",MULTIPLIER); + // TODO: check for existence of overflow file? + ReplayCharSequence rcs = ros.getReplayCharSequence(ENCODING); + + // Now check that start of the rcs comes back in as nihongo string. + String rcsStr = rcs.subSequence(0, nihongo.length()).toString(); + assertTrue("Nihongo " + nihongo + " does not equal converted string" + + " from rcs " + rcsStr, + nihongo.equals(rcsStr)); + // And assert next string is also properly nihongo. + if (rcs.length() >= (nihongo.length() * 2)) { + rcsStr = rcs.subSequence(nihongo.length(), + nihongo.length() + nihongo.length()).toString(); + assertTrue("Nihongo " + nihongo + " does not equal converted " + + " string from rcs (2nd time)" + rcsStr, + nihongo.equals(rcsStr)); + } + } + + public void testGetReplayCharSequenceByteZeroOffset() throws IOException { + + RecordingOutputStream ros = writeTestStream( + regularBuffer,MULTIPLIER, + "testGetReplayCharSequenceByteZeroOffset",MULTIPLIER); + ReplayCharSequence rcs = ros.getReplayCharSequence(); + + for (int i = 0; i < MULTIPLIER; i++) { + accessingCharacters(rcs); + } + } + + public void testGetReplayCharSequenceByteOffset() throws IOException { + + RecordingOutputStream ros = writeTestStream( + regularBuffer,MULTIPLIER, + "testGetReplayCharSequenceByteOffset",MULTIPLIER); + ReplayCharSequence rcs = ros.getReplayCharSequence(null,SEQUENCE_LENGTH); + + for (int i = 0; i < MULTIPLIER; i++) { + accessingCharacters(rcs); + } + } + + public void testGetReplayCharSequenceMultiByteZeroOffset() + throws IOException { + + RecordingOutputStream ros = writeTestStream( + regularBuffer,MULTIPLIER, + "testGetReplayCharSequenceMultiByteZeroOffset",MULTIPLIER); + ReplayCharSequence rcs = ros.getReplayCharSequence("UTF-8"); + + for (int i = 0; i < MULTIPLIER; i++) { + accessingCharacters(rcs); + } + } + + public void testGetReplayCharSequenceMultiByteOffset() throws IOException { + + RecordingOutputStream ros = writeTestStream( + regularBuffer,MULTIPLIER, + "testGetReplayCharSequenceMultiByteOffset",MULTIPLIER); + ReplayCharSequence rcs = ros.getReplayCharSequence("UTF-8", SEQUENCE_LENGTH); + + try { + for (int i = 0; i < MULTIPLIER; i++) { + accessingCharacters(rcs); + } + } finally { + rcs.close(); + } + } + + public void testReplayCharSequenceByteToString() throws IOException { + String fileContent = "Some file content"; + byte [] buffer = fileContent.getBytes(); + RecordingOutputStream ros = writeTestStream( + buffer,1, + "testReplayCharSequenceByteToString.txt",0); + ReplayCharSequence rcs = ros.getReplayCharSequence(); + String result = rcs.toString(); + assertEquals("Strings don't match",result,fileContent); + } + + public void testReplayCharSequenceByteToStringOverflow() throws IOException { + String fileContent = "Some file content. "; + byte [] buffer = fileContent.getBytes(); + RecordingOutputStream ros = writeTestStream( + buffer,1, + "testReplayCharSequenceByteToString.txt",1); + String expectedContent = fileContent+fileContent; + ReplayCharSequence rcs = ros.getReplayCharSequence(); + String result = rcs.toString(); + assertEquals("Strings don't match", expectedContent, result); + } + + public void testReplayCharSequenceByteToStringMulti() throws IOException { + String fileContent = "Some file content"; + byte [] buffer = fileContent.getBytes("UTF-8"); + final int MULTIPLICAND = 10; + StringBuilder sb = + new StringBuilder(MULTIPLICAND * fileContent.length()); + for (int i = 0; i < MULTIPLICAND; i++) { + sb.append(fileContent); + } + String expectedResult = sb.toString(); + RecordingOutputStream ros = writeTestStream( + buffer,1, + "testReplayCharSequenceByteToStringMulti.txt",MULTIPLICAND-1); + for (int i = 0; i < 3; i++) { + ReplayCharSequence rcs = ros.getReplayCharSequence("UTF-8"); + String result = rcs.toString(); + assertEquals("Strings don't match", result, expectedResult); + rcs.close(); + System.gc(); + System.runFinalization(); + } + } + + /** + * Accessing characters test. + * + * Checks that characters in the rcs are in sequence. + * + * @param rcs The ReplayCharSequence to try out. + */ + private void accessingCharacters(CharSequence rcs) { + long timestamp = (new Date()).getTime(); + int seeks = 0; + for (int i = (INCREMENT * 2); (i + INCREMENT) < rcs.length(); + i += INCREMENT) { + checkCharacter(rcs, i); + seeks++; + for (int j = i - INCREMENT; j < i; j++) { + checkCharacter(rcs, j); + seeks++; + } + } + // Note that printing out below breaks cruisecontrols drawing + // of the xml unit test results because it outputs disallowed + // xml characters. + logger.fine(rcs + " seeks count " + seeks + " in " + + ((new Date().getTime()) - timestamp) + " milliseconds."); + } + + /** + * Check the character read. + * + * Throws assertion if not expected result. + * + * @param rcs ReplayCharSequence to read from. + * @param i Character offset. + */ + private void checkCharacter(CharSequence rcs, int i) { + int c = rcs.charAt(i); + assertTrue("Character " + Integer.toString(c) + " at offset " + i + + " unexpected.", (c % SEQUENCE_LENGTH) == (i % SEQUENCE_LENGTH)); + } + + /** + * @param baseName + * @return RecordingOutputStream + * @throws IOException + */ + private RecordingOutputStream writeTestStream(byte[] content, + int memReps, String baseName, int fileReps) throws IOException { + String backingFilename = FileUtils.maybeRelative(getTmpDir(),baseName).getAbsolutePath(); + RecordingOutputStream ros = new RecordingOutputStream( + content.length * memReps, + backingFilename); + ros.open(); + for(int i = 0; i < (memReps+fileReps); i++) { + // fill buffer (repeat MULTIPLIER times) and + // overflow to disk (also MULTIPLIER times) + ros.write(content); + } + ros.close(); + return ros; + } + + + /** + * Fill a buffer w/ regular progression of single-byte + * (and <= 127) characters. + * @param buffer Buffer to fill. + * @return The buffer we filled. + */ + private byte [] fillBufferWithRegularContent(byte [] buffer) { + int index = 0; + for (int i = 0; i < buffer.length; i++) { + buffer[i] = (byte) (index & 0x00ff); + index++; + if (index >= SEQUENCE_LENGTH) { + // Reset the index. + index = 0; + } + } + return buffer; + } + + public void testCheckParameters() + { + // TODO. + } +} diff --git a/commons/src/test/java/org/archive/io/RepositionableInputStreamTest.java b/commons/src/test/java/org/archive/io/RepositionableInputStreamTest.java new file mode 100644 index 00000000..ad76ee4b --- /dev/null +++ b/commons/src/test/java/org/archive/io/RepositionableInputStreamTest.java @@ -0,0 +1,76 @@ +/* RepositionableInputStreamTest.java + * + * $Id$ + * + * Created Dec 20, 2005 + * + * Copyright (C) 2005 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.io; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.PrintWriter; + +import org.archive.util.TmpDirTestCase; + +public class RepositionableInputStreamTest extends TmpDirTestCase { + private File testFile; + private static final String LINE = "0123456789abcdefghijklmnopqrstuv"; + protected void setUp() throws Exception { + super.setUp(); + this.testFile = new File(getTmpDir(), this.getClass().getName()); + PrintWriter pw = new PrintWriter(new FileOutputStream(testFile)); + for (int i = 0; i < 100; i++) { + pw.print(LINE); + } + pw.close(); + } + protected void tearDown() throws Exception { + super.tearDown(); + } + public void testname() throws Exception { + // Make buffer awkward size so we run into buffers spanning issues. + RepositionableInputStream ris = + new RepositionableInputStream(new FileInputStream(this.testFile), + 57); + int c = ris.read(); + assertEquals(1, ris.position()); + ris.read(); + ris.position(0); + assertEquals(0, ris.position()); + int c1 = ris.read(); + assertEquals(c, c1); + ris.position(0); + byte [] bytes = new byte[LINE.length()]; + long offset = 0; + for (int i = 0; i < 10; i++) { + ris.read(bytes, 0, LINE.length()); + assertEquals(LINE, new String(bytes)); + offset += LINE.length(); + assertEquals(offset, ris.position()); + } + long p = ris.position(); + ris.position(p - LINE.length()); + assertEquals(p - LINE.length(), ris.position()); + c = ris.read(); + assertEquals(c, c1); + } +} diff --git a/commons/src/test/java/org/archive/io/SinkHandlerTest.java b/commons/src/test/java/org/archive/io/SinkHandlerTest.java new file mode 100644 index 00000000..9e993f69 --- /dev/null +++ b/commons/src/test/java/org/archive/io/SinkHandlerTest.java @@ -0,0 +1,84 @@ +/* SinkHandlerTest.java + * + * $Id$ + * + * Created Aug 9, 2005 + * + * Copyright (C) 2005 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.io; + +import junit.framework.TestCase; + +public class SinkHandlerTest extends TestCase { + + public void testNothing() { + } + +// Commented out the below test code because it, like so many other +// things that are good in the world, does not work with maven2. + +// maven2 doesn't usually use the system class loader when executing +// tests, and the below code requires the system class loader for +// loading the LogHandler (per LogHandler restrictions). + +// We could configure maven2 to use the system class loader, but then +// we would have to use the same classpath as maven2. Unfortunately +// there's a conflict: maven2 uses commons-lang-2.1, and we require +// commons-lang-2.3. + +/* protected void setUp() throws Exception { + super.setUp(); + Class.forName("org.archive.io.SinkHandler"); + String logConfig = "handlers = " + + "org.archive.io.SinkHandler\n" + + "org.archive.io.SinkHandler.level = ALL"; + ByteArrayInputStream bais = + new ByteArrayInputStream(logConfig.getBytes()); + LogManager.getLogManager().readConfiguration(bais); + } + + public void testLogging() throws Exception { + LogRecord lr = new LogRecord(Level.SEVERE, ""); + long base = lr.getSequenceNumber() + 1; + System.out.println(base); + LOGGER.severe("Test1"); + LOGGER.severe("Test2"); + LOGGER.warning("Test3"); + RuntimeException e = new RuntimeException("Nothing exception"); + LOGGER.log(Level.SEVERE, "with exception", e); + SinkHandler h = SinkHandler.getInstance(); + assertEquals(h.getAllUnread().size(), 4); + SinkHandlerLogRecord shlr = h.get(base + 3); + assertTrue(shlr != null); + h.remove(base + 3); + assertEquals(h.getAllUnread().size(), 3); + h.publish(shlr); + assertEquals(h.getAllUnread().size(), 4); + } */ + /* + public void testToString() throws Exception { + RuntimeException e = new RuntimeException("Some-Message"); + LOGGER.log(Level.SEVERE, "With-Exception", e); + SinkHandler h = SinkHandler.getInstance(); + System.out.print(((SeenLogRecord)h.getSink().get(0)).toString()); + LOGGER.log(Level.SEVERE, "No-Exception"); + System.out.print(((SeenLogRecord)h.getSink().get(1)).toString()); + }*/ +} diff --git a/commons/src/test/java/org/archive/io/arc/ARCReaderFactoryTest.java b/commons/src/test/java/org/archive/io/arc/ARCReaderFactoryTest.java new file mode 100644 index 00000000..ffabcb9d --- /dev/null +++ b/commons/src/test/java/org/archive/io/arc/ARCReaderFactoryTest.java @@ -0,0 +1,97 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io.arc; + +import java.io.File; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Iterator; + +import org.archive.io.ArchiveRecord; +import org.archive.util.TmpDirTestCase; + +public class ARCReaderFactoryTest extends TmpDirTestCase { +// public void testGetHttpURL() throws MalformedURLException, IOException { +// ARCReader reader = null; +// try { +// // TODO: I can get a single ARCRecord but trying to iterate from +// // a certain point is getting an EOR when I go to read GZIP header. +// reader = ARCReaderFactory. +// get(new URL("http://localhost/test.arc.gz"), 0); +// for (final Iterator i = reader.iterator(); i.hasNext();) { +// ARCRecord ar = (ARCRecord)i.next(); +// System.out.println(ar.getMetaData().getUrl()); +// } +// } finally { +// if (reader != null) { +// reader.close(); +// } +// } +// } + + /** + * Test File URL. + * If a file url, we just use the pointed to file. There is no + * copying down to a file in tmp that gets cleaned up after close. + * @throws MalformedURLException + * @throws IOException + */ + public void testGetFileURL() throws MalformedURLException, IOException { + File arc = ARCWriterTest.createARCFile(getTmpDir(), true); + doGetFileUrl(arc); + } + + protected void doGetFileUrl(File arc) + throws MalformedURLException, IOException { + ARCReader reader = null; + File tmpFile = null; + try { + reader = ARCReaderFactory. + get(new URL("file:////" + arc.getAbsolutePath())); + tmpFile = null; + for (Iterator i = reader.iterator(); i.hasNext();) { + ARCRecord r = (ARCRecord)i.next(); + if (tmpFile == null) { + tmpFile = new File(r.getMetaData().getArc()); + } + } + assertTrue(tmpFile.exists()); + } finally { + if (reader != null) { + reader.close(); + } + } + assertTrue(tmpFile.exists()); + } + + /** + * Test path or url. + * @throws MalformedURLException + * @throws IOException + */ + public void testGetPathOrURL() throws MalformedURLException, IOException { + File arc = ARCWriterTest.createARCFile(getTmpDir(), true); + ARCReader reader = ARCReaderFactory.get(arc.getAbsoluteFile()); + assertNotNull(reader); + reader.close(); + doGetFileUrl(arc); + } +} diff --git a/commons/src/test/java/org/archive/io/arc/ARCWriterPoolTest.java b/commons/src/test/java/org/archive/io/arc/ARCWriterPoolTest.java new file mode 100644 index 00000000..b0425fc8 --- /dev/null +++ b/commons/src/test/java/org/archive/io/arc/ARCWriterPoolTest.java @@ -0,0 +1,158 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io.arc; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.NoSuchElementException; + +import org.archive.io.WriterPoolMember; +import org.archive.io.WriterPool; +import org.archive.io.WriterPoolSettings; +import org.archive.util.TmpDirTestCase; + + +/** + * Test ARCWriterPool + */ +@SuppressWarnings("deprecation") +public class ARCWriterPoolTest extends TmpDirTestCase { + private static final String PREFIX = "TEST"; + + public void testARCWriterPool() + throws Exception { + final int MAX_ACTIVE = 3; + final int MAX_WAIT_MILLISECONDS = 100; + cleanUpOldFiles(PREFIX); + WriterPool pool = new ARCWriterPool(getSettings(true), + MAX_ACTIVE, MAX_WAIT_MILLISECONDS); + WriterPoolMember [] writers = new WriterPoolMember[MAX_ACTIVE]; + final String CONTENT = "Any old content"; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + baos.write(CONTENT.getBytes()); + for (int i = 0; i < MAX_ACTIVE; i++) { + writers[i] = pool.borrowFile(); + assertEquals("Number active", i + 1, pool.getNumActive()); + ((ARCWriter)writers[i]).write("http://one.two.three", "no-type", + "0.0.0.0", 1234567890, CONTENT.length(), baos); + } + + // Pool is maxed out. Try and get a new ARCWriter. We'll block for + // MAX_WAIT_MILLISECONDS. Should get exception. + long start = (new Date()).getTime(); + boolean isException = false; + try { + pool.borrowFile(); + } catch(NoSuchElementException e) { + isException = true; + long end = (new Date()).getTime(); + // This test can fail on a loaded machine if the wait period is + // only MAX_WAIT_MILLISECONDS. Up the time to wait. + final int WAIT = MAX_WAIT_MILLISECONDS * 100; + if ((end - start) > (WAIT)) { + fail("More than " + MAX_WAIT_MILLISECONDS + " elapsed: " + + WAIT); + } + } + assertTrue("Did not get NoSuchElementException", isException); + + for (int i = (MAX_ACTIVE - 1); i >= 0; i--) { + pool.returnFile(writers[i]); + assertEquals("Number active", i, pool.getNumActive()); + assertEquals("Number idle", MAX_ACTIVE - pool.getNumActive(), + pool.getNumIdle()); + } + pool.close(); + } + + public void testInvalidate() throws Exception { + final int MAX_ACTIVE = 3; + final int MAX_WAIT_MILLISECONDS = 100; + cleanUpOldFiles(PREFIX); + WriterPool pool = new ARCWriterPool(getSettings(true), + MAX_ACTIVE, MAX_WAIT_MILLISECONDS); + WriterPoolMember [] writers = new WriterPoolMember[MAX_ACTIVE]; + final String CONTENT = "Any old content"; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + baos.write(CONTENT.getBytes()); + for (int i = 0; i < MAX_ACTIVE; i++) { + writers[i] = pool.borrowFile(); + assertEquals("Number active", i + 1, pool.getNumActive()); + ((ARCWriter)writers[i]).write("http://one.two.three", "no-type", + "0.0.0.0", 1234567890, CONTENT.length(), baos); + } + + WriterPoolMember writer2Invalidate = writers[pool.getNumActive() - 1]; + writers[pool.getNumActive() - 1] = null; + pool.invalidateFile(writer2Invalidate); + for (int i = 0; i < (MAX_ACTIVE - 1); i++) { + if (writers[i] == null) { + continue; + } + pool.returnFile(writers[i]); + } + + for (int i = 0; i < MAX_ACTIVE; i++) { + writers[i] = pool.borrowFile(); + assertEquals("Number active", i + 1, pool.getNumActive()); + ((ARCWriter)writers[i]).write("http://one.two.three", "no-type", + "0.0.0.0", 1234567890, CONTENT.length(), baos); + } + for (int i = (MAX_ACTIVE - 1); i >= 0; i--) { + pool.returnFile(writers[i]); + assertEquals("Number active", i, pool.getNumActive()); + assertEquals("Number idle", MAX_ACTIVE - pool.getNumActive(), + pool.getNumIdle()); + } + pool.close(); + } + + private WriterPoolSettings getSettings(final boolean isCompressed) { + return new WriterPoolSettings() { + public long getMaxSize() { + return ARCConstants.DEFAULT_MAX_ARC_FILE_SIZE; + } + + public String getPrefix() { + return PREFIX; + } + + public String getSuffix() { + return ""; + } + + public List getOutputDirs() { + File [] files = {getTmpDir()}; + return Arrays.asList(files); + } + + public boolean isCompressed() { + return isCompressed; + } + + public List getMetadata() { + return null; + } + }; + } +} \ No newline at end of file diff --git a/commons/src/test/java/org/archive/io/arc/ARCWriterTest.java b/commons/src/test/java/org/archive/io/arc/ARCWriterTest.java new file mode 100644 index 00000000..8f1f7583 --- /dev/null +++ b/commons/src/test/java/org/archive/io/arc/ARCWriterTest.java @@ -0,0 +1,558 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io.arc; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.PrintStream; +import java.util.Arrays; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.input.NullInputStream; +import org.apache.commons.io.output.NullOutputStream; +import org.archive.io.ArchiveRecord; +import org.archive.io.ArchiveRecordHeader; +import org.archive.io.ReplayInputStream; +import org.archive.io.WriterPoolMember; +import org.archive.util.ArchiveUtils; +import org.archive.util.FileUtils; +import org.archive.util.TmpDirTestCase; + + +/** + * Test ARCWriter class. + * + * This code exercises ARCWriter AND ARCReader. First it writes ARCs w/ + * ARCWriter. Then it validates what was written w/ ARCReader. + * + * @author stack + */ +public class ARCWriterTest +extends TmpDirTestCase implements ARCConstants { + /** + * Utility class for writing bad ARCs (with trailing junk) + */ + public class CorruptibleARCWriter extends ARCWriter { + byte[] endJunk = null; + + public CorruptibleARCWriter(AtomicInteger serial_no, List name, + String name2, boolean compress, long default_max_arc_file_size) { + super(serial_no, name, name2, compress, default_max_arc_file_size); + } + + @Override + protected void postWriteRecordTasks() throws IOException { + if (endJunk != null) { + this.write(endJunk); + } + super.postWriteRecordTasks(); + } + + public void setEndJunk(byte[] b) throws IOException { + this.endJunk = b; + } + } + + /** + * Prefix to use for ARC files made by JUNIT. + */ + private static final String SUFFIX = + /* TODO DEFAULT_ARC_FILE_PREFIX*/ "JUNIT"; + + private static final String SOME_URL = "http://www.archive.org/test/"; + + + private static final AtomicInteger SERIAL_NO = new AtomicInteger(); + + /* + * @see TestCase#setUp() + */ + protected void setUp() throws Exception { + super.setUp(); + } + + /* + * @see TestCase#tearDown() + */ + protected void tearDown() throws Exception { + super.tearDown(); + } + + protected static String getContent() { + return getContent(null); + } + + protected static String getContent(String indexStr) { + String page = (indexStr != null)? "Page #" + indexStr: "Some Page"; + return "HTTP/1.1 200 OK\r\n" + + "Content-Type: text/html\r\n\r\n" + + "" + page + + "" + + "" + page + + ""; + } + + @SuppressWarnings("deprecation") + protected int writeRandomHTTPRecord(ARCWriter arcWriter, int index) + throws IOException { + String indexStr = Integer.toString(index); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + // Start the record with an arbitrary 14-digit date per RFC2540 + String now = ArchiveUtils.get14DigitDate(); + int recordLength = 0; + byte[] record = (getContent(indexStr)).getBytes(); + recordLength += record.length; + baos.write(record); + // Add the newline between records back in + baos.write("\n".getBytes()); + recordLength += 1; + arcWriter.write("http://www.one.net/id=" + indexStr, "text/html", + "0.1.2.3", Long.parseLong(now), recordLength, baos); + return recordLength; + } + + private File writeRecords(String baseName, boolean compress, + long maxSize, int recordCount) + throws IOException { + cleanUpOldFiles(baseName); + File [] files = {getTmpDir()}; + ARCWriter arcWriter = new ARCWriter(SERIAL_NO, Arrays.asList(files), + baseName + '-' + SUFFIX, compress, maxSize); + assertNotNull(arcWriter); + for (int i = 0; i < recordCount; i++) { + writeRandomHTTPRecord(arcWriter, i); + } + arcWriter.close(); + assertTrue("Doesn't exist: " + + arcWriter.getFile().getAbsolutePath(), + arcWriter.getFile().exists()); + return arcWriter.getFile(); + } + + private void validate(File arcFile, int recordCount) + throws FileNotFoundException, IOException { + ARCReader reader = ARCReaderFactory.get(arcFile); + assertNotNull(reader); + List metaDatas = null; + if (recordCount == -1) { + metaDatas = reader.validate(); + } else { + metaDatas = reader.validate(recordCount); + } + reader.close(); + // Now, run through each of the records doing absolute get going from + // the end to start. Reopen the arc so no context between this test + // and the previous. + reader = ARCReaderFactory.get(arcFile); + for (int i = metaDatas.size() - 1; i >= 0; i--) { + ARCRecordMetaData meta = (ARCRecordMetaData)metaDatas.get(i); + ArchiveRecord r = reader.get(meta.getOffset()); + String mimeType = r.getHeader().getMimetype(); + assertTrue("Record is bogus", + mimeType != null && mimeType.length() > 0); + } + reader.close(); + assertTrue("Metadatas not equal", metaDatas.size() == recordCount); + for (Iterator i = metaDatas.iterator(); i.hasNext();) { + ARCRecordMetaData r = (ARCRecordMetaData)i.next(); + assertTrue("Record is empty", r.getLength() > 0); + } + } + + public void testCheckARCFileSize() + throws IOException { + runCheckARCFileSizeTest("checkARCFileSize", false); + } + + public void testCheckARCFileSizeCompressed() + throws IOException { + runCheckARCFileSizeTest("checkARCFileSize", true); + } + + public void testWriteRecord() throws IOException { + final int recordCount = 2; + File arcFile = writeRecords("writeRecord", false, + DEFAULT_MAX_ARC_FILE_SIZE, recordCount); + validate(arcFile, recordCount + 1); // Header record. + } + + public void testRandomAccess() throws IOException { + final int recordCount = 3; + File arcFile = writeRecords("writeRecord", true, + DEFAULT_MAX_ARC_FILE_SIZE, recordCount); + ARCReader reader = ARCReaderFactory.get(arcFile); + // Get to second record. Get its offset for later use. + boolean readFirst = false; + String url = null; + long offset = -1; + long totalRecords = 0; + boolean readSecond = false; + for (final Iterator i = reader.iterator(); i.hasNext(); totalRecords++) { + ARCRecord ar = (ARCRecord)i.next(); + if (!readFirst) { + readFirst = true; + continue; + } + if (!readSecond) { + url = ar.getMetaData().getUrl(); + offset = ar.getMetaData().getOffset(); + readSecond = true; + } + } + + reader = ARCReaderFactory.get(arcFile, offset); + ArchiveRecord ar = reader.get(); + assertEquals(ar.getHeader().getUrl(), url); + ar.close(); + + // Get reader again. See how iterator works with offset + reader = ARCReaderFactory.get(arcFile, offset); + int count = 0; + for (final Iterator i = reader.iterator(); i.hasNext(); i.next()) { + count++; + } + reader.close(); + assertEquals(totalRecords - 1, count); + } + + public void testWriteRecordCompressed() throws IOException { + final int recordCount = 2; + File arcFile = writeRecords("writeRecordCompressed", true, + DEFAULT_MAX_ARC_FILE_SIZE, recordCount); + validate(arcFile, recordCount + 1 /*Header record*/); + } + + public void testWriteGiantRecord() throws IOException { + PrintStream dummyStream = new PrintStream(new NullOutputStream()); + ARCWriter arcWriter = new ARCWriter(SERIAL_NO, dummyStream, + new File("dummy"), + false, null, null); + assertNotNull(arcWriter); + + // Start the record with an arbitrary 14-digit date per RFC2540 + long now = System.currentTimeMillis(); + long recordLength = org.apache.commons.io.FileUtils.ONE_GB * 3; + + arcWriter.write("dummy:uri", "application/octet-stream", + "0.1.2.3", now, recordLength, new NullInputStream(recordLength)); + arcWriter.close(); + } + + private void runCheckARCFileSizeTest(String baseName, boolean compress) + throws FileNotFoundException, IOException { + writeRecords(baseName, compress, 1024, 15); + // Now validate all files just created. + File [] files = FileUtils.getFilesWithPrefix(getTmpDir(), SUFFIX); + for (int i = 0; i < files.length; i++) { + validate(files[i], -1); + } + } + + protected CorruptibleARCWriter createARCWriter(String NAME, boolean compress) { + File [] files = {getTmpDir()}; + return new CorruptibleARCWriter(SERIAL_NO, Arrays.asList(files), NAME, + compress, DEFAULT_MAX_ARC_FILE_SIZE); + } + + protected static ByteArrayInputStream getBais(String str) + throws IOException { + return new ByteArrayInputStream(str.getBytes()); + } + + /** + * Writes a record, suppressing normal length-checks (so that + * intentionally malformed records may be written). + */ + protected static void writeRecord(ARCWriter writer, String url, + String type, int len, ByteArrayInputStream bais) + throws IOException { + writer.write(url, type, "192.168.1.1", (new Date()).getTime(), len, + bais, false); + } + + protected int iterateRecords(ARCReader r) + throws IOException { + int count = 0; + for (Iterator i = r.iterator(); i.hasNext();) { + ARCRecord rec = (ARCRecord)i.next(); + rec.close(); + if (count != 0) { + assertTrue("Unexpected URL " + rec.getMetaData().getUrl(), + rec.getMetaData().getUrl().equals(SOME_URL)); + } + count++; + } + return count; + } + + protected CorruptibleARCWriter createArcWithOneRecord(String name, + boolean compressed) + throws IOException { + CorruptibleARCWriter writer = createARCWriter(name, compressed); + String content = getContent(); + writeRecord(writer, SOME_URL, "text/html", + content.length(), getBais(content)); + return writer; + } + + public void testSpaceInURL() { + String eMessage = null; + try { + holeyUrl("testSpaceInURL-" + SUFFIX, false, " "); + } catch (IOException e) { + eMessage = e.getMessage(); + } + assertTrue("Didn't get expected exception: " + eMessage, + eMessage.startsWith("Metadata line doesn't match")); + } + + public void testTabInURL() { + String eMessage = null; + try { + holeyUrl("testTabInURL-" + SUFFIX, false, "\t"); + } catch (IOException e) { + eMessage = e.getMessage(); + } + assertTrue("Didn't get expected exception: " + eMessage, + eMessage.startsWith("Metadata line doesn't match")); + } + + protected void holeyUrl(String name, boolean compress, String urlInsert) + throws IOException { + ARCWriter writer = createArcWithOneRecord(name, compress); + // Add some bytes on the end to mess up the record. + String content = getContent(); + writeRecord(writer, SOME_URL + urlInsert + "/index.html", "text/html", + content.length(), getBais(content)); + writer.close(); + } + +// If uncompressed, length has to be right or parse will fail. +// +// public void testLengthTooShort() throws IOException { +// lengthTooShort("testLengthTooShort-" + PREFIX, false); +// } + + public void testLengthTooShortCompressed() throws IOException { + lengthTooShort("testLengthTooShortCompressed-" + SUFFIX, true, false); + } + + public void testLengthTooShortCompressedStrict() + throws IOException { + String eMessage = null; + try { + lengthTooShort("testLengthTooShortCompressedStrict-" + SUFFIX, + true, true); + } catch (RuntimeException e) { + eMessage = e.getMessage(); + } + assertTrue("Didn't get expected exception: " + eMessage, + eMessage.startsWith("java.io.IOException: Record ENDING at")); + } + + protected void lengthTooShort(String name, boolean compress, boolean strict) + throws IOException { + CorruptibleARCWriter writer = createArcWithOneRecord(name, compress); + // Add some bytes on the end to mess up the record. + String content = getContent(); + ByteArrayInputStream bais = getBais(content+"SOME TRAILING BYTES"); + writeRecord(writer, SOME_URL, "text/html", + content.length(), bais); + writer.setEndJunk("SOME TRAILING BYTES".getBytes()); + writeRecord(writer, SOME_URL, "text/html", + content.length(), getBais(content)); + writer.close(); + + // Catch System.err into a byte stream. + ByteArrayOutputStream os = new ByteArrayOutputStream(); + System.setErr(new PrintStream(os)); + + ARCReader r = ARCReaderFactory.get(writer.getFile()); + r.setStrict(strict); + int count = iterateRecords(r); + assertTrue("Count wrong " + count, count == 4); + + // Make sure we get the warning string which complains about the + // trailing bytes. + String err = os.toString(); + assertTrue("No message " + err, err.startsWith("WARNING") && + (err.indexOf("Record ENDING at") > 0)); + } + +// If uncompressed, length has to be right or parse will fail. +// +// public void testLengthTooLong() +// throws IOException { +// lengthTooLong("testLengthTooLongCompressed-" + PREFIX, +// false, false); +// } + + public void testLengthTooLongCompressed() + throws IOException { + lengthTooLong("testLengthTooLongCompressed-" + SUFFIX, + true, false); + } + + public void testLengthTooLongCompressedStrict() { + String eMessage = null; + try { + lengthTooLong("testLengthTooLongCompressed-" + SUFFIX, + true, true); + } catch (IOException e) { + eMessage = e.getMessage(); + } + assertTrue("Didn't get expected exception: " + eMessage, + eMessage.startsWith("Premature EOF before end-of-record")); + } + + protected void lengthTooLong(String name, boolean compress, + boolean strict) + throws IOException { + ARCWriter writer = createArcWithOneRecord(name, compress); + // Add a record with a length that is too long. + String content = getContent(); + writeRecord(writer, SOME_URL, "text/html", + content.length() + 10, getBais(content)); + writeRecord(writer, SOME_URL, "text/html", + content.length(), getBais(content)); + writer.close(); + + // Catch System.err. + ByteArrayOutputStream os = new ByteArrayOutputStream(); + System.setErr(new PrintStream(os)); + + ARCReader r = ARCReaderFactory.get(writer.getFile()); + r.setStrict(strict); + int count = iterateRecords(r); + assertTrue("Count wrong " + count, count == 4); + + // Make sure we get the warning string which complains about the + // trailing bytes. + String err = os.toString(); + assertTrue("No message " + err, + err.startsWith("WARNING Premature EOF before end-of-record")); + } + + public void testGapError() throws IOException { + ARCWriter writer = createArcWithOneRecord("testGapError", true); + String content = getContent(); + // Make a 'weird' RIS that returns bad 'remaining' length + // awhen remaining should be 0 + ReplayInputStream ris = new ReplayInputStream(content.getBytes(), + content.length(), null) { + public long remaining() { + return (super.remaining()==0) ? -1 : super.remaining(); + } + }; + String message = null; + try { + writer.write(SOME_URL, "text/html", "192.168.1.1", + (new Date()).getTime(), content.length(), ris); + } catch (IOException e) { + message = e.getMessage(); + } finally { + IOUtils.closeQuietly(ris); + } + writer.close(); + assertTrue("No gap when should be", + message != null && + message.indexOf("Gap between expected and actual") >= 0); + } + + /** + * Write an arc file for other tests to use. + * @param arcdir Directory to write to. + * @param compress True if file should be compressed. + * @return ARC written. + * @throws IOException + */ + public static File createARCFile(File arcdir, boolean compress) + throws IOException { + File [] files = {arcdir}; + ARCWriter writer = new ARCWriter(SERIAL_NO, Arrays.asList(files), + "test", compress, DEFAULT_MAX_ARC_FILE_SIZE); + String content = getContent(); + writeRecord(writer, SOME_URL, "text/html", content.length(), + getBais(content)); + writer.close(); + return writer.getFile(); + } + +// public void testSpeed() throws IOException { +// ARCWriter writer = createArcWithOneRecord("speed", true); +// // Add a record with a length that is too long. +// String content = getContent(); +// final int count = 100000; +// logger.info("Starting speed write of " + count + " records."); +// for (int i = 0; i < count; i++) { +// writeRecord(writer, SOME_URL, "text/html", content.length(), +// getBaos(content)); +// } +// writer.close(); +// logger.info("Finished speed write test."); +// } + + + public void testValidateMetaLine() throws Exception { + final String line = "http://www.aandw.net/images/walden2.png " + + "128.197.34.86 20060111174224 image/png 2160"; + ARCWriter w = createARCWriter("testValidateMetaLine", true); + try { + w.validateMetaLine(line); + w.validateMetaLine(line + LINE_SEPARATOR); + w.validateMetaLine(line + "\\r\\n"); + } finally { + w.close(); + } + } + + public void testArcRecordOffsetReads() throws Exception { + // Get an ARC with one record. + WriterPoolMember w = + createArcWithOneRecord("testArcRecordInBufferStream", true); + w.close(); + // Get reader on said ARC. + ARCReader r = ARCReaderFactory.get(w.getFile()); + final Iterator i = r.iterator(); + // Skip first ARC meta record. + ARCRecord ar = (ARCRecord) i.next(); + i.hasNext(); + // Now we're at first and only record in ARC. + ar = (ARCRecord) i.next(); + // Now try getting some random set of bytes out of it + // at an odd offset (used to fail because we were + // doing bad math to find where in buffer to read). + final byte[] buffer = new byte[17]; + final int maxRead = 4; + int totalRead = 0; + while (totalRead < maxRead) { + totalRead = totalRead + + ar.read(buffer, 13 + totalRead, maxRead - totalRead); + assertTrue(totalRead > 0); + } + } +} diff --git a/commons/src/test/java/org/archive/net/PublicSuffixesTest.java b/commons/src/test/java/org/archive/net/PublicSuffixesTest.java new file mode 100644 index 00000000..4be57141 --- /dev/null +++ b/commons/src/test/java/org/archive/net/PublicSuffixesTest.java @@ -0,0 +1,113 @@ +/* PublicSuffixesTest.java + * + * $Id: BloomFilter32bitSplit.java 5197 2007-06-06 01:31:46Z gojomo $ + * + * Created on Jun 13, 2007 + * + * Copyright (C) 2007 Internet Archive + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +package org.archive.net; + +import java.util.regex.Matcher; + +import junit.framework.TestCase; + +/** + * Test cases for PublicSuffixes utility. Confirm expected matches/nonmatches + * from constructed regex. + * + * @author gojomo + */ +public class PublicSuffixesTest extends TestCase { + Matcher m = PublicSuffixes.getTopmostAssignedSurtPrefixPattern() + .matcher(""); + + public void testBasics() { + matchPrefix("com,example,www,", "com,example,"); + matchPrefix("com,example,", "com,example,"); + matchPrefix("org,archive,www,", "org,archive,"); + matchPrefix("org,archive,", "org,archive,"); + matchPrefix("fr,yahoo,www,", "fr,yahoo,"); + matchPrefix("fr,yahoo,", "fr,yahoo,"); + matchPrefix("au,com,foobar,www,", "au,com,foobar,"); + matchPrefix("au,com,foobar,", "au,com,foobar,"); + matchPrefix("uk,co,virgin,www,", "uk,co,virgin,"); + matchPrefix("uk,co,virgin,", "uk,co,virgin,"); + matchPrefix("au,com,example,www,", "au,com,example,"); + matchPrefix("au,com,example,", "au,com,example,"); + matchPrefix("jp,tokyo,public,assigned,www,", + "jp,tokyo,public,assigned,"); + matchPrefix("jp,tokyo,public,assigned,", "jp,tokyo,public,assigned,"); + } + + public void testDomainWithDash() { + matchPrefix("de,bad-site,www", "de,bad-site,"); + } + + public void testDomainWithNumbers() { + matchPrefix("de,archive4u,www", "de,archive4u,"); + } + + public void testIPV4() { + assertEquals("unexpected reduction", + "1.2.3.4", + PublicSuffixes.reduceSurtToTopmostAssigned("1.2.3.4")); + } + + public void testIPV6() { + assertEquals("unexpected reduction", + "[2001:0db8:85a3:08d3:1319:8a2e:0370:7344]", + PublicSuffixes.reduceSurtToTopmostAssigned( + "[2001:0db8:85a3:08d3:1319:8a2e:0370:7344]")); + } + + public void testExceptions() { + matchPrefix("uk,bl,www,", "uk,bl,"); + matchPrefix("uk,bl,", "uk,bl,"); + matchPrefix("jp,tokyo,metro,subdomain,", "jp,tokyo,metro,"); + matchPrefix("jp,tokyo,metro,", "jp,tokyo,metro,"); + } + + public void testFakeTLD() { + // we assume any new/unknonwn TLD should be assumed as 2-level; + // this is preferable for our grouping purpose but might not be + // for a cookie-assigning browser (original purpose of publicsuffixlist) + matchPrefix("zzz,example,www,", "zzz,example,"); + } + + public void testUnsegmentedHostname() { + m.reset("example"); + assertFalse("unexpected match found in 'example'", m.find()); + } + + public void testTopmostAssignedCaching() { + assertSame("topmostAssignedSurtPrefixPattern not cached",PublicSuffixes.getTopmostAssignedSurtPrefixPattern(),PublicSuffixes.getTopmostAssignedSurtPrefixPattern()); + assertSame("topmostAssignedSurtPrefixRegex not cached",PublicSuffixes.getTopmostAssignedSurtPrefixRegex(),PublicSuffixes.getTopmostAssignedSurtPrefixRegex()); + } + + // TODO: test UTF domains? + + protected void matchPrefix(String surtDomain, String expectedAssignedPrefix) { + m.reset(surtDomain); + assertTrue("expected match not found in '" + surtDomain, m.find()); + assertEquals("expected match not found", expectedAssignedPrefix, m + .group()); + } +} diff --git a/commons/src/test/java/org/archive/net/UURIFactoryTest.java b/commons/src/test/java/org/archive/net/UURIFactoryTest.java new file mode 100644 index 00000000..de1acd59 --- /dev/null +++ b/commons/src/test/java/org/archive/net/UURIFactoryTest.java @@ -0,0 +1,1102 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.net; + +import java.util.Iterator; +import java.util.TreeMap; + +import junit.framework.TestCase; + +import org.apache.commons.httpclient.URIException; +import org.apache.commons.lang.SerializationUtils; + +/** + * Test UURIFactory for proper UURI creation across variety of + * important/tricky cases. + * + * Be careful writing this file. Make sure you write it with UTF-8 encoding. + * + * @author igor stack gojomo + */ +public class UURIFactoryTest extends TestCase { + + public final void testEscaping() throws URIException { + // Note: single quote is not being escaped by URI class. + final String ESCAPED_URISTR = "http://archive.org/" + + UURIFactory.ESCAPED_SPACE + + UURIFactory.ESCAPED_SPACE + + UURIFactory.ESCAPED_CIRCUMFLEX + + UURIFactory.ESCAPED_QUOT + + UURIFactory.SQUOT + + UURIFactory.ESCAPED_APOSTROPH + + UURIFactory.ESCAPED_LSQRBRACKET + + UURIFactory.ESCAPED_RSQRBRACKET + + UURIFactory.ESCAPED_LCURBRACKET + + UURIFactory.ESCAPED_RCURBRACKET + + UURIFactory.SLASH + "a.gif"; // NBSP and SPACE should be trimmed; + + final String URISTR = "http://archive.org/.././" + "\u00A0" + + UURIFactory.SPACE + UURIFactory.CIRCUMFLEX + + UURIFactory.QUOT + UURIFactory.SQUOT + + UURIFactory.APOSTROPH + UURIFactory.LSQRBRACKET + + UURIFactory.RSQRBRACKET + UURIFactory.LCURBRACKET + + UURIFactory.RCURBRACKET + UURIFactory.BACKSLASH + + "test/../a.gif" + "\u00A0" + UURIFactory.SPACE; + + UURI uuri = UURIFactory.getInstance(URISTR); + final String uuriStr = uuri.toString(); + assertEquals("expected escaping", ESCAPED_URISTR, uuriStr); + } + + public final void testUnderscoreMakesPortParseFail() throws URIException { + UURI uuri = UURIFactory.getInstance("http://one-two_three:8080/index.html"); + int port = uuri.getPort(); + assertTrue("Failed find of port " + uuri, port == 8080); + } + + public final void testRelativeURIWithTwoSlashes() throws URIException { + UURI base = UURIFactory.getInstance("http://www.archive.org"); + UURI uuri = UURIFactory.getInstance(base, "one//index.html"); + assertTrue("Doesn't do right thing with two slashes " + uuri, + uuri.toString().equals( + "http://www.archive.org/one//index.html")); + } + + public final void testTrailingEncodedSpace() throws URIException { + UURI uuri = UURIFactory.getInstance("http://www.nps-shoes.co.uk%20"); + assertTrue("Doesn't strip trailing encoded space 1 " + uuri, + uuri.toString().equals("http://www.nps-shoes.co.uk/")); + uuri = UURIFactory.getInstance("http://www.nps-shoes.co.uk%20%20%20"); + assertTrue("Doesn't strip trailing encoded space 2 " + uuri, + uuri.toString().equals("http://www.nps-shoes.co.uk/")); + } + + public final void testPort0080is80() throws URIException { + UURI uuri = UURIFactory.getInstance("http://archive.org:0080"); + assertTrue("Doesn't strip leading zeros " + uuri, + uuri.toString().equals("http://archive.org/")); + } + +// DISABLING TEST AS PRECURSOR TO ELIMINATION +// the problematic input given -- specifically the "%6s" incomplete uri-escape, +// shouldn't necessarily be rejected as a bad URI. IE and Firefox, at least, +// will attempt to fetch such an URL (getting, in this case against that ad +// server, a bad-request error). Ideally, we'd generate exactly the same +// request against the server as they do. However, with the most recent +// fixup for stray '%' signs, we come close, but not exactly. That's enough +// to cause this test to fail (it's not getting the expected exception) but +// our almost-URI, which might be what was intended, is better than trying +// nothing. +// public final void testBadPath() { +// String message = null; +// try { +// UURIFactory.getInstance("http://ads.as4x.tmcs.net/" + +// "html.ng/site=cs&pagepos=102&page=home&adsize=1x1&context=" + +// "generic&Params.richmedia=yes%26city%3Dseattle%26" + +// "rstid%3D2415%26market_id%3D86%26brand%3Dcitysearch" + +// "%6state%3DWA"); +// } catch (URIException e) { +// message = e.getMessage(); +// } +// assertNotNull("Didn't get expected exception.", message); +// } + + public final void testEscapeEncoding() throws URIException { + UURI uuri = UURIFactory.getInstance("http://www.y1y1.com/" + + "albums/userpics/11111/normal_%E3%E4%EC%EC%EC.jpg", "windows-1256"); + uuri.getPath(); + } + + public final void testTooLongAfterEscaping() { + StringBuffer buffer = new StringBuffer("http://www.archive.org/a/"); + // Append bunch of spaces. When escaped, they'll triple in size. + for (int i = 0; i < 1024; i++) { + buffer.append(" "); + } + buffer.append("/index.html"); + String message = null; + try { + UURIFactory.getInstance(buffer.toString()); + } catch (URIException e) { + message = e.getMessage(); + } + assertTrue("Wrong or no exception: " + message, (message != null) && + message.startsWith("Created (escaped) uuri >")); + } + + public final void testFtpUris() throws URIException { + final String FTP = "ftp"; + final String AUTHORITY = "pfbuser:pfbuser@mprsrv.agri.gov.cn"; + final String PATH = "/clzreceive/"; + final String uri = FTP + "://" + AUTHORITY + PATH; + UURI uuri = UURIFactory.getInstance(uri); + assertTrue("Failed to get matching scheme: " + uuri.getScheme(), + (uuri.getScheme()).equals(FTP)); + assertTrue("Failed to get matching authority: " + + uuri.getAuthority(), (uuri.getAuthority()).equals(AUTHORITY)); + assertTrue("Failed to get matching path: " + + uuri.getPath(), (uuri.getPath()).equals(PATH)); + } + + public final void testWhitespaceEscaped() throws URIException { + // Test that we get all whitespace even if the uri is + // already escaped. + String uri = "http://archive.org/index%25 .html"; + String tgtUri = "http://archive.org/index%25%20.html"; + UURI uuri = UURIFactory.getInstance(uri); + assertTrue("Not equal " + uuri.toString(), + uuri.toString().equals(tgtUri)); + uri = "http://archive.org/index%25\u001D.html"; + tgtUri = "http://archive.org/index%25%1D.html".toLowerCase(); + uuri = UURIFactory.getInstance(uri); + assertEquals("whitespace escaping", tgtUri, uuri.toString()); + uri = "http://gemini.info.usaid.gov/directory/" + + "pbResults.cfm?&urlNameLast=Rumplestiltskin"; + tgtUri = "http://gemini.info.usaid.gov/directory/faxResults.cfm?" + + "name=Ebenezer%20+Rumplestiltskin,&location=RRB%20%20%20%205%2E08%2D006"; + uuri = UURIFactory.getInstance(UURIFactory.getInstance(uri), + "faxResults.cfm?name=Ebenezer +Rumplestiltskin,&location=" + + "RRB%20%20%20%205%2E08%2D006"); + assertEquals("whitespace escaping", tgtUri, uuri.toString()); + } + +// public final void testFailedGetPath() throws URIException { +// final String path = "/RealMedia/ads/" + +// "click_lx.ads/%%PAGE%%/%%RAND%%/%%POS%%/%%CAMP%%/empty"; +// // decoding in getPath will interpret %CA as 8-bit escaped char, +// // possibly incomplete +// final String uri = "http://ads.nandomedia.com" + path; +// final UURI uuri = UURIFactory.getInstance(uri); +// String foundPath = uuri.getPath(); +// assertEquals("unexpected path", path, foundPath); +// } + + public final void testDnsHost() throws URIException { + String uri = "dns://ads.nandomedia.com:81/one.html"; + UURI uuri = UURIFactory.getInstance(uri); + String host = uuri.getReferencedHost(); + assertTrue("Host is wrong " + host, host.equals("ads.nandomedia.com")); + uri = "dns:ads.nandomedia.com"; + uuri = UURIFactory.getInstance(uri); + host = uuri.getReferencedHost(); + assertTrue("Host is wrong " + host, host.equals("ads.nandomedia.com")); + uri = "dns:ads.nandomedia.com?a=b"; + uuri = UURIFactory.getInstance(uri); + host = uuri.getReferencedHost(); + assertTrue("Host is wrong " + host, host.equals("ads.nandomedia.com")); + } + + public final void testPercentEscaping() throws URIException { + final String uri = "http://archive.org/%a%%%%%.html"; + // tests indicate firefox (1.0.6) does not encode '%' at all + final String tgtUri = "http://archive.org/%a%%%%%.html"; + UURI uuri = UURIFactory.getInstance(uri); + assertEquals("Not equal",tgtUri, uuri.toString()); + } + + public final void testRelativeDblPathSlashes() throws URIException { + UURI base = UURIFactory.getInstance("http://www.archive.org/index.html"); + UURI uuri = UURIFactory.getInstance(base, "JIGOU//KYC//INDEX.HTM"); + assertTrue("Double slash not working " + uuri.toString(), + uuri.getPath().equals("/JIGOU//KYC//INDEX.HTM")); + } + + public final void testRelativeWithScheme() throws URIException { + UURI base = UURIFactory.getInstance("http://www.example.com/some/page"); + UURI uuri = UURIFactory.getInstance(base, "http:boo"); + assertTrue("Relative with scheme not working " + uuri.toString(), + uuri.toString().equals("http://www.example.com/some/boo")); + } + + public final void testBadBaseResolve() throws URIException { + UURI base = UURIFactory.getInstance("http://license.joins.com/board/" + + "etc_board_list.asp?board_name=new_main&b_type=&nPage=" + + "2&category=G&lic_id=70&site=changeup&g_page=changeup&g_sPage=" + + "notice&gate=02"); + UURIFactory.getInstance(base, "http://www.changeup.com/...[ 1010966 ] crawl.log has URIs with spaces in them. + * @throws URIException + */ + public final void testSpaceDoubleEncoding() throws URIException { + final String uri = "http://www.brook.edu/i.html? %20taxonomy=Politics"; + final String encodedUri = + "http://www.brook.edu/i.html?%20%20taxonomy=Politics"; + UURI uuri = UURIFactory.getInstance(uri, "ISO-8859-1"); + assertTrue("Not equal " + uuri.toString(), + uuri.toString().equals(encodedUri)); + } + + /** + * Test for doubly-encoded sequences. + * See [ 966219 ] UURI doubly-encodes %XX sequences. + * @throws URIException + */ + public final void testDoubleEncoding() throws URIException { + final char ae = '\u00E6'; + final String uri = "http://archive.org/DIR WITH SPACES/home" + + ae + ".html"; + final String encodedUri = + "http://archive.org/DIR%20WITH%20SPACES/home%E6.html"; + UURI uuri = UURIFactory.getInstance(uri, "ISO-8859-1"); + assertEquals("single encoding", encodedUri, uuri.toString()); + // Dbl-encodes. + uuri = UURIFactory.getInstance(uuri.toString(), "ISO-8859-1"); + uuri = UURIFactory.getInstance(uuri.toString(), "ISO-8859-1"); + assertEquals("double encoding", encodedUri, uuri.toString()); + // Do default utf-8 test. + uuri = UURIFactory.getInstance(uri); + final String encodedUtf8Uri = + "http://archive.org/DIR%20WITH%20SPACES/home%C3%A6.html"; + assertEquals("Not equal utf8", encodedUtf8Uri, uuri.toString()); + // Now dbl-encode. + uuri = UURIFactory.getInstance(uuri.toString()); + uuri = UURIFactory.getInstance(uuri.toString()); + assertEquals("Not equal (dbl-encoding) utf8", encodedUtf8Uri, uuri.toString()); + } + + /** + * Test for syntax errors stop page parsing. + * @see [ 788219 ] URI Syntax Errors stop page parsing + * @throws URIException + */ + public final void testThreeSlashes() throws URIException { + UURI goodURI = UURIFactory. + getInstance("http://lcweb.loc.gov/rr/goodtwo.html"); + String uuri = "http:///lcweb.loc.gov/rr/goodtwo.html"; + UURI rewrittenURI = UURIFactory.getInstance(uuri); + assertTrue("Not equal " + goodURI + ", " + uuri, + goodURI.toString().equals(rewrittenURI.toString())); + uuri = "http:////lcweb.loc.gov/rr/goodtwo.html"; + rewrittenURI = UURIFactory.getInstance(uuri); + assertTrue("Not equal " + goodURI + ", " + uuri, + goodURI.toString().equals(rewrittenURI.toString())); + // Check https. + goodURI = UURIFactory. + getInstance("https://lcweb.loc.gov/rr/goodtwo.html"); + uuri = "https:////lcweb.loc.gov/rr/goodtwo.html"; + rewrittenURI = UURIFactory.getInstance(uuri); + assertTrue("Not equal " + goodURI + ", " + uuri, + goodURI.toString().equals(rewrittenURI.toString())); + } + + public final void testNoScheme() { + boolean expectedException = false; + String uuri = "www.loc.gov/rr/european/egw/polishex.html"; + try { + UURIFactory.getInstance(uuri); + } catch (URIException e) { + // Expected exception. + expectedException = true; + } + assertTrue("Didn't get expected exception: " + uuri, + expectedException); + } + + public final void testRelative() throws URIException { + UURI uuriTgt = UURIFactory. + getInstance("http://archive.org:83/home.html"); + UURI uri = UURIFactory. + getInstance("http://archive.org:83/one/two/three.html"); + UURI uuri = UURIFactory. + getInstance(uri, "/home.html"); + assertTrue("Not equal", + uuriTgt.toString().equals(uuri.toString())); + } + + /** + * Test that an empty uuri does the right thing -- that we get back the + * base. + * + * @throws URIException + */ + public final void testRelativeEmpty() throws URIException { + UURI uuriTgt = UURIFactory. + getInstance("http://archive.org:83/one/two/three.html"); + UURI uri = UURIFactory. + getInstance("http://archive.org:83/one/two/three.html"); + UURI uuri = UURIFactory. + getInstance(uri, ""); + assertTrue("Empty length don't work", + uuriTgt.toString().equals(uuri.toString())); + } + + public final void testAbsolute() throws URIException { + UURI uuriTgt = UURIFactory. + getInstance("http://archive.org:83/home.html"); + UURI uri = UURIFactory. + getInstance("http://archive.org:83/one/two/three.html"); + UURI uuri = UURIFactory. + getInstance(uri, "http://archive.org:83/home.html"); + assertTrue("Not equal", + uuriTgt.toString().equals(uuri.toString())); + } + + /** + * Test for [ 962892 ] UURI accepting/creating unUsable URIs (bad hosts). + * @see [ 962892 ] UURI accepting/creating unUsable URIs (bad hosts) + */ + public final void testHostWithLessThan() { + checkExceptionOnIllegalDomainlabel("http://www.betamobile.com[ 1012520 ] UURI.length() > 2k + */ + public final void test2kURI() throws URIException { + final StringBuffer buffer = new StringBuffer("http://a.b"); + final String subPath = "/123456789"; + for (int i = 0; i < 207; i++) { + buffer.append(subPath); + } + // String should be 2080 characters long. Legal. + UURIFactory.getInstance(buffer.toString()); + boolean gotException = false; + // Add ten more characters and make size illegal. + buffer.append(subPath); + try { + UURIFactory.getInstance(buffer.toString()); + } catch (URIException e) { + gotException = true; + } + assertTrue("No expected exception complaining about long URI", + gotException); + } + + private void checkExceptionOnIllegalDomainlabel(String uuri) { + boolean expectedException = false; + try { + UURIFactory.getInstance(uuri); + } catch (URIException e) { + // Expected exception. + expectedException = true; + } + assertTrue("Didn't get expected exception: " + uuri, + expectedException); + } + + /** + * Test for doing separate DNS lookup for same host + * + * @see [ 788277 ] Doing separate DNS lookup for same host + * @throws URIException + */ + public final void testHostWithPeriod() throws URIException { + UURI uuri1 = UURIFactory. + getInstance("http://www.loc.gov./index.html"); + UURI uuri2 = UURIFactory. + getInstance("http://www.loc.gov/index.html"); + assertEquals("Failed equating hosts with dot", + uuri1.getHost(), uuri2.getHost()); + } + + /** + * Test for NPE in java.net.URI.encode + * + * @see [ 874220 ] NPE in java.net.URI.encode + * @throws URIException + */ + public final void testHostEncodedChars() throws URIException { + String s = "http://g.msn.co.kr/0nwkokr0/00/19??" + + "PS=10274&NC=10009&CE=42&CP=949&HL=" + + "���?��"; + assertNotNull("Encoded chars " + s, + UURIFactory.getInstance(s)); + } + + /** + * Test for java.net.URI parses %20 but getHost null + * + * See [ 927940 ] java.net.URI parses %20 but getHost null + */ + public final void testSpaceInHost() { + boolean expectedException = false; + try { + UURIFactory.getInstance( + "http://www.local-regions.odpm%20.gov.uk" + + "/lpsa/challenge/pdf/propect.pdf"); + } catch (URIException e) { + expectedException = true; + } + assertTrue("Did not fail with escaped space.", expectedException); + + expectedException = false; + try { + UURIFactory.getInstance( + "http://www.local-regions.odpm .gov.uk" + + "/lpsa/challenge/pdf/propect.pdf"); + } catch (URIException e) { + expectedException = true; + } + assertTrue("Did not fail with real space.", expectedException); + } + + /** + * Test for java.net.URI chokes on hosts_with_underscores. + * + * @see [ 808270 ] java.net.URI chokes on hosts_with_underscores + * @throws URIException + */ + public final void testHostWithUnderscores() throws URIException { + UURI uuri = UURIFactory.getInstance( + "http://x_underscore_underscore.2u.com.tw/nonexistent_page.html"); + assertEquals("Failed get of host with underscore", + "x_underscore_underscore.2u.com.tw", uuri.getHost()); + } + + + /** + * Two dots for igor. + */ + public final void testTwoDots() { + boolean expectedException = false; + try { + UURIFactory.getInstance( + "http://x_underscore_underscore..2u.com/nonexistent_page.html"); + } catch (URIException e) { + expectedException = true; + } + assertTrue("Two dots did not throw exception", expectedException); + } + + /** + * Test for java.net.URI#getHost fails when leading digit. + * + * @see [ 910120 ] java.net.URI#getHost fails when leading digit. + * @throws URIException + */ + public final void testHostWithDigit() throws URIException { + UURI uuri = UURIFactory. + getInstance("http://0204chat.2u.com.tw/nonexistent_page.html"); + assertEquals("Failed get of host with digit", + "0204chat.2u.com.tw", uuri.getHost()); + } + + /** + * Test for Constraining java URI class. + * + * @see [ 949548 ] Constraining java URI class + */ + public final void testPort() { + checkBadPort("http://www.tyopaikat.com:a/robots.txt"); + checkBadPort("http://158.144.21.3:80808/robots.txt"); + checkBadPort("http://pdb.rutgers.edu:81.rutgers.edu/robots.txt"); + checkBadPort( + "https://webmail.gse.harvard.edu:9100robots.txt/robots.txt"); + checkBadPort( + "https://webmail.gse.harvard.edu:0/robots.txt/robots.txt"); + } + + /** + * Test bad port throws exception. + * @param uri URI with bad port to check. + */ + private void checkBadPort(String uri) { + boolean exception = false; + try { + UURIFactory.getInstance(uri); + } + catch (URIException e) { + exception = true; + } + assertTrue("Didn't throw exception: " + uri, exception); + } + + /** + * Preserve userinfo capitalization. + * @throws URIException + */ + public final void testUserinfo() throws URIException { + final String authority = "stack:StAcK@www.tyopaikat.com"; + final String uri = "http://" + authority + "/robots.txt"; + UURI uuri = UURIFactory.getInstance(uri); + assertEquals("Authority not equal", uuri.getAuthority(), + authority); + /* + String tmp = uuri.toString(); + assertTrue("URI not equal", tmp.equals(uri)); + */ + } + + /** + * Test user info + port + * @throws URIException + */ + public final void testUserinfoPlusPort() throws URIException { + final String userInfo = "stack:StAcK"; + final String authority = "www.tyopaikat.com"; + final int port = 8080; + final String uri = "http://" + userInfo + "@" + authority + ":" + port + + "/robots.txt"; + UURI uuri = UURIFactory.getInstance(uri); + assertEquals("Host not equal", authority,uuri.getHost()); + assertEquals("Userinfo Not equal",userInfo,uuri.getUserinfo()); + assertEquals("Port not equal",port,uuri.getPort()); + assertEquals("Authority wrong","stack:StAcK@www.tyopaikat.com:8080", + uuri.getAuthority()); + assertEquals("AuthorityMinusUserinfo wrong","www.tyopaikat.com:8080", + uuri.getAuthorityMinusUserinfo()); + + } + + public final void testRFC3986RelativeChange() throws URIException { + UURI base = UURIFactory.getInstance("http://a/b/c/d;p?q"); + tryRelative(base, "?y", "http://a/b/c/d;p?y"); + } + + /** + * Tests from rfc3986 + * + * 

+     *       "g:h"           =  "g:h"
+     *       "g"             =  "http://a/b/c/g"
+     *       "./g"           =  "http://a/b/c/g"
+     *       "g/"            =  "http://a/b/c/g/"
+     *       "/g"            =  "http://a/g"
+     *       "//g"           =  "http://g"
+     *       "?y"            =  "http://a/b/c/d;p?y"
+     *       "g?y"           =  "http://a/b/c/g?y"
+     *       "#s"            =  "http://a/b/c/d;p?q#s"
+     *       "g#s"           =  "http://a/b/c/g#s"
+     *       "g?y#s"         =  "http://a/b/c/g?y#s"
+     *       ";x"            =  "http://a/b/c/;x"
+     *       "g;x"           =  "http://a/b/c/g;x"
+     *       "g;x?y#s"       =  "http://a/b/c/g;x?y#s"
+     *       ""              =  "http://a/b/c/d;p?q"
+     *       "."             =  "http://a/b/c/"
+     *       "./"            =  "http://a/b/c/"
+     *       ".."            =  "http://a/b/"
+     *       "../"           =  "http://a/b/"
+     *       "../g"          =  "http://a/b/g"
+     *       "../.."         =  "http://a/"
+     *       "../../"        =  "http://a/"
+     *       "../../g"       =  "http://a/g"
+     *
+ * + * @throws URIException + */ + public final void testRFC3986Relative() throws URIException { + UURI base = UURIFactory.getInstance("http://a/b/c/d;p?q"); + tryRelative(base, "g:h", "g:h"); + tryRelative(base, "g", "http://a/b/c/g"); + tryRelative(base, "./g", "http://a/b/c/g"); + tryRelative(base, "g/", "http://a/b/c/g/"); + tryRelative(base, "/g", "http://a/g"); + tryRelative(base, "//g", "http://g"); + tryRelative(base, "?y", "http://a/b/c/d;p?y"); + tryRelative(base, "g?y", "http://a/b/c/g?y"); + tryRelative(base, "#s", "http://a/b/c/d;p?q#s"); + tryRelative(base, "g#s", "http://a/b/c/g#s"); + tryRelative(base, "g?y#s", "http://a/b/c/g?y#s"); + tryRelative(base, ";x", "http://a/b/c/;x"); + tryRelative(base, "g;x", "http://a/b/c/g;x"); + tryRelative(base, "g;x?y#s","http://a/b/c/g;x?y#s"); + tryRelative(base, "", "http://a/b/c/d;p?q"); + tryRelative(base, ".", "http://a/b/c/"); + tryRelative(base, "./", "http://a/b/c/"); + tryRelative(base, "..", "http://a/b/"); + tryRelative(base, "../", "http://a/b/"); + tryRelative(base, "../g", "http://a/b/g"); + tryRelative(base, "../..", "http://a/"); + tryRelative(base, "../../", "http://a/"); + tryRelative(base, "../../g","http://a/g"); + } + + protected void tryRelative(UURI base, String relative, String expected) + throws URIException { + UURI uuri = UURIFactory.getInstance(base, relative); + assertEquals("Derelativized " + relative + " gave " + + uuri + " not " + expected, + UURIFactory.getInstance(expected),uuri); + } + + /** + * Tests from rfc2396 with amendments to accomodate differences + * intentionally added to make our URI handling like IEs. + * + * 
+	 *       g:h           =  g:h
+	 *       g             =  http://a/b/c/g
+	 *       ./g           =  http://a/b/c/g
+	 *       g/            =  http://a/b/c/g/
+	 *       /g            =  http://a/g
+	 *       //g           =  http://g
+	 *       ?y            =  http://a/b/c/?y
+	 *       g?y           =  http://a/b/c/g?y
+	 *       #s            =  (current document)#s
+	 *       g#s           =  http://a/b/c/g#s
+	 *       g?y#s         =  http://a/b/c/g?y#s
+	 *       ;x            =  http://a/b/c/;x
+	 *       g;x           =  http://a/b/c/g;x
+	 *       g;x?y#s       =  http://a/b/c/g;x?y#s
+	 *       .             =  http://a/b/c/
+	 *       ./            =  http://a/b/c/
+	 *       ..            =  http://a/b/
+	 *       ../           =  http://a/b/
+	 *       ../g          =  http://a/b/g
+	 *       ../..         =  http://a/
+	 *       ../../        =  http://a/
+	 *       ../../g       =  http://a/g
+	 *
+ * + * @throws URIException + */ + public final void testRFC2396Relative() throws URIException { + UURI base = UURIFactory. + getInstance("http://a/b/c/d;p?q"); + TreeMap m = new TreeMap(); + m.put("..", "http://a/b/"); + m.put("../", "http://a/b/"); + m.put("../g", "http://a/b/g"); + m.put("../..", "http://a/"); + m.put("../../", "http://a/"); + m.put("../../g", "http://a/g"); + m.put("g#s", "http://a/b/c/g#s"); + m.put("g?y#s ", "http://a/b/c/g?y#s"); + m.put(";x", "http://a/b/c/;x"); + m.put("g;x", "http://a/b/c/g;x"); + m.put("g;x?y#s", "http://a/b/c/g;x?y#s"); + m.put(".", "http://a/b/c/"); + m.put("./", "http://a/b/c/"); + m.put("g", "http://a/b/c/g"); + m.put("./g", "http://a/b/c/g"); + m.put("g/", "http://a/b/c/g/"); + m.put("/g", "http://a/g"); + m.put("//g", "http://g"); + // CHANGED BY RFC3986 + // m.put("?y", "http://a/b/c/?y"); + m.put("g?y", "http://a/b/c/g?y"); + // EXTRAS beyond the RFC set. + // TODO: That these resolve to a path of /a/g might be wrong. Perhaps + // it should be '/g'?. + m.put("/../../../../../../../../g", "http://a/g"); + m.put("../../../../../../../../g", "http://a/g"); + m.put("../G", "http://a/b/G"); + for (Iterator i = m.keySet().iterator(); i.hasNext();) { + String key = (String)i.next(); + String value = (String)m.get(key); + UURI uuri = UURIFactory.getInstance(base, key); + assertTrue("Unexpected " + key + " " + value + " " + uuri, + uuri.equals(UURIFactory.getInstance(value))); + } + } + + /** + * A UURI should always be without a 'fragment' segment, which is + * unused and irrelevant for network fetches. + * + * See [ 970666 ] #anchor links not trimmed, and thus recrawled + * + * @throws URIException + */ + public final void testAnchors() throws URIException { + UURI uuri = UURIFactory. + getInstance("http://www.example.com/path?query#anchor"); + assertEquals("Not equal", "http://www.example.com/path?query", + uuri.toString()); + } + + + /** + * Ensure that URI strings beginning with a colon are treated + * the same as browsers do (as relative, rather than as absolute + * with zero-length scheme). + * + * @throws URIException + */ + public void testStartsWithColon() throws URIException { + UURI base = UURIFactory.getInstance("http://www.example.com/path/page"); + UURI uuri = UURIFactory.getInstance(base,":foo"); + assertEquals("derelativize starsWithColon", + uuri.getURI(), + "http://www.example.com/path/:foo"); + } + + /** + * Ensure that stray trailing '%' characters do not prevent + * UURI instances from being created, and are reasonably + * escaped when encountered. + * + * @throws URIException + */ + public void testTrailingPercents() throws URIException { + String plainPath = "http://www.example.com/path%"; + UURI plainPathUuri = UURIFactory.getInstance(plainPath); + assertEquals("plainPath getURI", plainPath, plainPathUuri.getURI()); + assertEquals("plainPath getEscapedURI", + "http://www.example.com/path%", // browsers don't escape '%' + plainPathUuri.getEscapedURI()); + + String partiallyEscapedPath = "http://www.example.com/pa%20th%"; + UURI partiallyEscapedPathUuri = UURIFactory.getInstance( + partiallyEscapedPath); +// assertEquals("partiallyEscapedPath getURI", +// "http://www.example.com/pa th%", // TODO: is this desirable? +//// partiallyEscapedPath, +// partiallyEscapedPathUuri.getURI()); + assertEquals("partiallyEscapedPath getEscapedURI", + "http://www.example.com/pa%20th%", + partiallyEscapedPathUuri.getEscapedURI()); + + String plainQueryString = "http://www.example.com/path?q=foo%"; + UURI plainQueryStringUuri = UURIFactory.getInstance( + plainQueryString); +// assertEquals("plainQueryString getURI", +// plainQueryString, +// plainQueryStringUuri.getURI()); + assertEquals("plainQueryString getEscapedURI", + "http://www.example.com/path?q=foo%", + plainQueryStringUuri.getEscapedURI()); + + String partiallyEscapedQueryString = + "http://www.example.com/pa%20th?q=foo%"; + UURI partiallyEscapedQueryStringUuri = UURIFactory.getInstance( + partiallyEscapedQueryString); + assertEquals("partiallyEscapedQueryString getURI", + "http://www.example.com/pa th?q=foo%", + partiallyEscapedQueryStringUuri.getURI()); + assertEquals("partiallyEscapedQueryString getEscapedURI", + "http://www.example.com/pa%20th?q=foo%", + partiallyEscapedQueryStringUuri.getEscapedURI()); + } + + /** + * Ensure that stray '%' characters do not prevent + * UURI instances from being created, and are reasonably + * escaped when encountered. + * + * @throws URIException + */ + public void testStrayPercents() throws URIException { + String oneStray = "http://www.example.com/pa%th"; + UURI oneStrayUuri = UURIFactory.getInstance(oneStray); + assertEquals("oneStray getURI", oneStray, oneStrayUuri.getURI()); + assertEquals("oneStray getEscapedURI", + "http://www.example.com/pa%th", // browsers don't escape '%' + oneStrayUuri.getEscapedURI()); + + String precededByValidEscape = "http://www.example.com/pa%20th%way"; + UURI precededByValidEscapeUuri = UURIFactory.getInstance( + precededByValidEscape); + assertEquals("precededByValidEscape getURI", + "http://www.example.com/pa th%way", // getURI interprets escapes + precededByValidEscapeUuri.getURI()); + assertEquals("precededByValidEscape getEscapedURI", + "http://www.example.com/pa%20th%way", + precededByValidEscapeUuri.getEscapedURI()); + + String followedByValidEscape = "http://www.example.com/pa%th%20way"; + UURI followedByValidEscapeUuri = UURIFactory.getInstance( + followedByValidEscape); + assertEquals("followedByValidEscape getURI", + "http://www.example.com/pa%th way", // getURI interprets escapes + followedByValidEscapeUuri.getURI()); + assertEquals("followedByValidEscape getEscapedURI", + "http://www.example.com/pa%th%20way", + followedByValidEscapeUuri.getEscapedURI()); + } + + public void testEscapingNotNecessary() throws URIException { + String escapesUnnecessary = + "http://www.example.com/misc;reserved:chars@that&don't=need" + +"+escaping$even,though!you(might)initially?think#so"; + // expect everything but the #fragment + String expected = escapesUnnecessary.substring(0, escapesUnnecessary + .length() - 3); + assertEquals("escapes unnecessary", + expected, + UURIFactory.getInstance(escapesUnnecessary).toString()); + } + + public void testIdn() throws URIException { + // See http://www.josefsson.org/idn.php. + // http://räksmörgÃ¥s.josefßon.org/ + String idn1 = "http://r\u00e4ksm\u00f6rg\u00e5s.josef\u00dfon.org/"; + String puny1 = "http://xn--rksmrgs-5wao1o.josefsson.org/"; + assertEquals("encoding of " + idn1, puny1, UURIFactory + .getInstance(idn1).toString()); + // http://www.pølse.dk/ + String idn2 = "http://www.p\u00f8lse.dk/"; + String puny2 = "http://www.xn--plse-gra.dk/"; + assertEquals("encoding of " + idn2, puny2, UURIFactory + .getInstance(idn2).toString()); + } + + public void testNewLineInURL() throws URIException { + UURI uuri = UURIFactory.getInstance("http://www.ar\rchive\n." + + "org/i\n\n\r\rndex.html"); + assertEquals("http://www.archive.org/index.html", uuri.toString()); + } + + public void testTabsInURL() throws URIException { + UURI uuri = UURIFactory.getInstance("http://www.ar\tchive\t." + + "org/i\t\r\n\tndex.html"); + assertEquals("http://www.archive.org/index.html", uuri.toString()); + } + + public void testQueryEscaping() throws URIException { + UURI uuri = UURIFactory.getInstance( + "http://www.yahoo.com/foo?somechars!@$%^&*()_-+={[}]|\'\";:/?.>,<"); + assertEquals( + // tests in FF1.5 indicate it only escapes " < > + "http://www.yahoo.com/foo?somechars!@$%^&*()_-+={[}]|\'%22;:/?.%3E,%3C", + uuri.toString()); + } + + /** + * Check that our 'normalization' does same as Nutch's + * Below before-and-afters were taken from the nutch urlnormalizer-basic + * TestBasicURLNormalizer class (December 2006, Nutch 0.9-dev). + * @throws URIException + */ + public void testSameAsNutchURLFilterBasic() throws URIException { + assertEquals(UURIFactory.getInstance(" http://foo.com/ ").toString(), + "http://foo.com/"); + + // check that protocol is lower cased + assertEquals(UURIFactory.getInstance("HTTP://foo.com/").toString(), + "http://foo.com/"); + + // check that host is lower cased + assertEquals(UURIFactory. + getInstance("http://Foo.Com/index.html").toString(), + "http://foo.com/index.html"); + assertEquals(UURIFactory. + getInstance("http://Foo.Com/index.html").toString(), + "http://foo.com/index.html"); + + // check that port number is normalized + assertEquals(UURIFactory. + getInstance("http://foo.com:80/index.html").toString(), + "http://foo.com/index.html"); + assertEquals(UURIFactory.getInstance("http://foo.com:81/").toString(), + "http://foo.com:81/"); + + // check that null path is normalized + assertEquals(UURIFactory.getInstance("http://foo.com").toString(), + "http://foo.com/"); + + // check that references are removed + assertEquals(UURIFactory. + getInstance("http://foo.com/foo.html#ref").toString(), + "http://foo.com/foo.html"); + + // // check that encoding is normalized + // normalizeTest("http://foo.com/%66oo.html", "http://foo.com/foo.html"); + + // check that unnecessary "../" are removed + assertEquals(UURIFactory. + getInstance("http://foo.com/aa/../").toString(), + "http://foo.com/" ); + assertEquals(UURIFactory. + getInstance("http://foo.com/aa/bb/../").toString(), + "http://foo.com/aa/"); + + /* We fail this one. Here we produce: 'http://foo.com/'. + assertEquals(UURIFactory. + getInstance("http://foo.com/aa/..").toString(), + "http://foo.com/aa/.."); + */ + + assertEquals(UURIFactory. + getInstance("http://foo.com/aa/bb/cc/../../foo.html").toString(), + "http://foo.com/aa/foo.html"); + assertEquals(UURIFactory. + getInstance("http://foo.com/aa/bb/../cc/dd/../ee/foo.html"). + toString(), + "http://foo.com/aa/cc/ee/foo.html"); + assertEquals(UURIFactory. + getInstance("http://foo.com/../foo.html").toString(), + "http://foo.com/foo.html" ); + assertEquals(UURIFactory. + getInstance("http://foo.com/../../foo.html").toString(), + "http://foo.com/foo.html" ); + assertEquals(UURIFactory. + getInstance("http://foo.com/../aa/../foo.html").toString(), + "http://foo.com/foo.html" ); + assertEquals(UURIFactory. + getInstance("http://foo.com/aa/../../foo.html").toString(), + "http://foo.com/foo.html" ); + assertEquals(UURIFactory. + getInstance("http://foo.com/aa/../bb/../foo.html/../../"). + toString(), + "http://foo.com/" ); + assertEquals(UURIFactory.getInstance("http://foo.com/../aa/foo.html"). + toString(), "http://foo.com/aa/foo.html" ); + assertEquals(UURIFactory. + getInstance("http://foo.com/../aa/../foo.html").toString(), + "http://foo.com/foo.html" ); + assertEquals(UURIFactory. + getInstance("http://foo.com/a..a/foo.html").toString(), + "http://foo.com/a..a/foo.html" ); + assertEquals(UURIFactory. + getInstance("http://foo.com/a..a/../foo.html").toString(), + "http://foo.com/foo.html" ); + assertEquals(UURIFactory. + getInstance("http://foo.com/foo.foo/../foo.html").toString(), + "http://foo.com/foo.html" ); + } + + public void testHttpSchemeColonSlash() { + boolean exception = false; + try { + UURIFactory.getInstance("https:/"); + } catch (URIException e) { + exception = true; + } + assertTrue("Didn't throw exception when one expected", exception); + exception = false; + try { + UURIFactory.getInstance("http://"); + } catch (URIException e) { + exception = true; + } + assertTrue("Didn't throw exception when one expected", exception); + } + + public void testNakedHttpsSchemeColon() { + boolean exception = false; + try { + UURIFactory.getInstance("https:"); + } catch (URIException e) { + exception = true; + } + assertTrue("Didn't throw exception when one expected", exception); + exception = false; + try { + UURI base = UURIFactory.getInstance("http://www.example.com"); + UURIFactory.getInstance(base, "https:"); + } catch (URIException e) { + exception = true; + } + assertTrue("Didn't throw exception when one expected", exception); + } + + /** + * Test motivated by [#HER-616] The UURI class may throw + * NullPointerException in getReferencedHost() + * + * @throws URIException + */ + public void testMissingHttpColon() throws URIException { + String suspectUri = "http//www.test.foo"; + UURI base = UURIFactory.getInstance("http://www.example.com"); + boolean exceptionThrown = false; + try { + UURI badUuri = UURIFactory.getInstance(suspectUri); + badUuri.getReferencedHost(); // not reached + } catch (URIException e) { + // should get relative-uri-no-base exception + exceptionThrown = true; + } finally { + assertTrue("expected exception not thrown",exceptionThrown); + } + UURI goodUuri = UURIFactory.getInstance(base,suspectUri); + goodUuri.getReferencedHost(); + } + + /** + * A UURI's string representation should be same after a + * serialization roundtrip. + * + * @throws URIException + */ + public final void testSerializationRoundtrip() throws URIException { + UURI uuri = UURIFactory. + getInstance("http://www.example.com/path?query#anchor"); + UURI uuri2 = (UURI) SerializationUtils.deserialize( + SerializationUtils.serialize(uuri)); + assertEquals("Not equal", uuri.toString(), uuri2.toString()); + uuri = UURIFactory. + getInstance("file://///boo_hoo/wwwroot/CMS/Images1/Banner.gif"); + uuri2 = (UURI) SerializationUtils.deserialize( + SerializationUtils.serialize(uuri)); + assertEquals("Not equal", uuri.toString(), uuri2.toString()); + } + + /** + * A UURI's string representation should be same after a + * toCustomString-getInstance roundtrip. + * + * @throws URIException + */ + public final void testToCustomStringRoundtrip() throws URIException { + UURI uuri = UURIFactory. + getInstance("http://www.example.com/path?query#anchor"); + UURI uuri2 = UURIFactory.getInstance(uuri.toCustomString()); + assertEquals("Not equal", uuri.toString(), uuri2.toString()); + // TODO: fix + // see [HER-1470] UURI String roundtrip (UURIFactory.getInstance(uuri.toString()) results in different URI for file: (and perhaps other) URIs + // http://webteam.archive.org/jira/browse/HER-1470 +// uuri = UURIFactory. +// getInstance("file://///boo_hoo/wwwroot/CMS/Images1/Banner.gif"); +// uuri2 = UURIFactory.getInstance(uuri.toCustomString()); +// assertEquals("Not equal", uuri.toString(), uuri2.toString()); + } +} diff --git a/commons/src/test/java/org/archive/net/UURITest.java b/commons/src/test/java/org/archive/net/UURITest.java new file mode 100644 index 00000000..22ac951a --- /dev/null +++ b/commons/src/test/java/org/archive/net/UURITest.java @@ -0,0 +1,50 @@ +/* UURITest.java + * + * $Id$ + * + * Created Jul 18, 2005 + * + * Copyright (C) 2005 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.net; + +import java.net.URISyntaxException; + +import junit.framework.TestCase; + +public class UURITest extends TestCase { + public void testHasScheme() { + assertTrue(UURI.hasScheme("http://www.archive.org")); + assertTrue(UURI.hasScheme("http:")); + assertFalse(UURI.hasScheme("ht/tp://www.archive.org")); + assertFalse(UURI.hasScheme("/tmp")); + } + + public void testGetFileName() throws URISyntaxException { + final String filename = "x.arc.gz"; + assertEquals(filename, + UURI.parseFilename("/tmp/one.two/" + filename)); + assertEquals(filename, + UURI.parseFilename("http://archive.org/tmp/one.two/" + + filename)); + assertEquals(filename, + UURI.parseFilename("rsync://archive.org/tmp/one.two/" + + filename)); + } +} diff --git a/commons/src/test/java/org/archive/settings/file/BdbModuleTest.java b/commons/src/test/java/org/archive/settings/file/BdbModuleTest.java new file mode 100644 index 00000000..eda23d90 --- /dev/null +++ b/commons/src/test/java/org/archive/settings/file/BdbModuleTest.java @@ -0,0 +1,99 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.settings.file; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.io.FileUtils; +import org.archive.bdb.BdbModule; +import org.archive.checkpointing.DefaultCheckpointRecovery; +import org.archive.spring.ConfigPath; +import org.archive.util.TmpDirTestCase; + +/** + * @author pjack + */ +public class BdbModuleTest extends TmpDirTestCase { + + public void testCheckpoint() throws Exception { + doCheckpoint(); + + File first = new File(getTmpDir(), "first"); +// File checkpointDir = new File(getTmpDir(), "checkpoint"); + + File second = new File(getTmpDir(), "second"); + FileUtils.deleteDirectory(second); + + DefaultCheckpointRecovery cr = new DefaultCheckpointRecovery("job"); + cr.getFileTranslations().put(first.getAbsolutePath(), + second.getAbsolutePath()); +// SheetManager mgr2 = Checkpointer.recover(checkpointDir, cr); +// BdbModule bdb2 = (BdbModule)mgr2.getRoot().get("module"); +// Map testData2 = bdb2.getBigMap("testData", false, +// String.class, String.class); + Map map1 = new HashMap(); + for (int i = 0; i < 1000; i++) { + map1.put(String.valueOf(i), String.valueOf(i * 2)); + } + +// Map map2 = dump(testData2); +// assertEquals(map1, map2); + } + + + private void doCheckpoint() throws Exception { + File first = new File(getTmpDir(), "first"); + FileUtils.deleteDirectory(first); + + File firstState = new File(first, "state"); +// MemorySheetManager mgr = new MemorySheetManager(); + + BdbModule bdb = new BdbModule(); +// mgr.getRoot().put("module", bdb); + bdb.setDir(new ConfigPath("test",firstState.getAbsolutePath())); +// mgr.getGlobalSheet().set(bdb, BdbModule.DIR, firstState.getAbsolutePath()); + bdb.start(); + + BdbModule.BdbConfig config = new BdbModule.BdbConfig(); + config.setAllowCreate(true); + bdb.openDatabase("testOpen", config, false); + + Map testData = bdb.getBigMap("testData", false, + String.class, String.class); + for (int i = 0; i < 1000; i++) { + testData.put(String.valueOf(i), String.valueOf(i * 2)); + } + + File checkpointDir = new File(getTmpDir(), "checkpoint"); + checkpointDir.mkdirs(); +// Checkpointer.checkpoint(mgr, checkpointDir); + bdb.stop(); + } + +// private Map dump(Map src) { +// HashMap dest = new HashMap(); +// for (String k: src.keySet()) { +// dest.put(k, src.get(k)); +// } +// return dest; +// } +} diff --git a/commons/src/test/java/org/archive/settings/file/PrefixFinderTest.java b/commons/src/test/java/org/archive/settings/file/PrefixFinderTest.java new file mode 100644 index 00000000..fa95ee12 --- /dev/null +++ b/commons/src/test/java/org/archive/settings/file/PrefixFinderTest.java @@ -0,0 +1,160 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.settings.file; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.apache.commons.io.FileUtils; +import org.archive.util.PrefixFinder; +import org.archive.util.TmpDirTestCase; + +import com.sleepycat.bind.tuple.StringBinding; +import com.sleepycat.collections.StoredSortedMap; +import com.sleepycat.je.Database; +import com.sleepycat.je.DatabaseConfig; +import com.sleepycat.je.Environment; +import com.sleepycat.je.EnvironmentConfig; + +/** + * Unit test for PrefixFinder. + * + * @author pjack + */ +public class PrefixFinderTest extends TmpDirTestCase { + + public void xtestFind() { + for (int i = 0; i < 100; i++) { + doTest(); + } + } + + public void testNoneFoundSmallSet() { + SortedSet testData = new TreeSet(); + testData.add("foo"); + List result = PrefixFinder.find(testData, "baz"); + assertTrue(result.isEmpty()); + } + + public void testOneFoundSmallSet() { + SortedSet testData = new TreeSet(); + testData.add("foo"); + List result = PrefixFinder.find(testData, "foobar"); + assertTrue(result.size()==1); + assertTrue(result.contains("foo")); + } + + public void testSortedMap() { + TreeMap map = new TreeMap(); + testUrlsNoMatch(map); + } + + @SuppressWarnings("unchecked") + public void testStoredSortedMap() throws Exception { + EnvironmentConfig config = new EnvironmentConfig(); + config.setAllowCreate(true); + config.setLockTimeout(5000000); + config.setCachePercent(5); + + File f = new File(getTmpDir(), "PrefixFinderText"); + FileUtils.deleteQuietly(f); + f.mkdirs(); + Environment bdbEnvironment = new Environment(f, config); + DatabaseConfig dbConfig = new DatabaseConfig(); + dbConfig.setAllowCreate(true); + dbConfig.setDeferredWrite(true); + Database db = bdbEnvironment.openDatabase(null, "test", dbConfig); + + StoredSortedMap ssm = new StoredSortedMap(db, new StringBinding(), new StringBinding(), true); + testUrlsNoMatch(ssm); + db.close(); + bdbEnvironment.close(); + } + + private void testUrlsNoMatch(SortedMap sm) { + sm.put("http://(com,ilovepauljack,www,", "foo"); + for (int i = 0; i < 10; i++) { + sm.put("http://" + Math.random(), "foo"); + } + Set keys = sm.keySet(); + if(!(keys instanceof SortedSet)) { + keys = new TreeSet(keys); + } + List results = PrefixFinder.find((SortedSet)keys, "http://"); + assertTrue(results.isEmpty()); + } + + private void doTest() { + // Generate test data. + SortedSet testData = new TreeSet(); + long seed = System.currentTimeMillis(); + System.out.println("Used seed: " + seed); + Random random = new Random(seed); + String prefix = "0"; + testData.add(prefix); + for (int i = 1; i < 10000; i++) { + if (random.nextInt(1024) == 0) { + prefix += " " + i; + testData.add(prefix); + } else { + testData.add(prefix + " " + i); + } + } + + // Brute-force to get the expected results. + List expected = new ArrayList(); + for (String value: testData) { + if (prefix.startsWith(value)) { + expected.add(value); + } + } + + // Results go from longest to shortest. + Collections.reverse(expected); + + final List result = PrefixFinder.find(testData, prefix); + + if (!result.equals(expected)) { + System.out.println("Expected: " + expected); + System.out.println("Result: " + result); + } + assertEquals(result, expected); + + // Double-check. + for (String value: result) { + if (!prefix.startsWith(value)) { + System.out.println("Result: " + result); + fail("Prefix string \"" + prefix + + "\" does not start with result key \"" + + value + "\""); + } + } + } + + +} diff --git a/commons/src/test/java/org/archive/surt/SURTTokenizerTest.java b/commons/src/test/java/org/archive/surt/SURTTokenizerTest.java new file mode 100644 index 00000000..62a90f96 --- /dev/null +++ b/commons/src/test/java/org/archive/surt/SURTTokenizerTest.java @@ -0,0 +1,191 @@ +/* SURTTokenizerTest + * + * $Id$ + * + * Created on 3:40:18 PM May 11, 2006. + * + * Copyright (C) 2006 Internet Archive. + * + * This file is part of wayback. + * + * wayback is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * wayback is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with wayback; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.surt; + +import junit.framework.TestCase; + +import org.apache.commons.httpclient.URIException; + +/** + * + * + * @author brad + * @version $Date$, $Revision$ + */ +public class SURTTokenizerTest extends TestCase { + + SURTTokenizer tok; + /** + * Test method for 'org.archive.wayback.accesscontrol.SURTTokenizer.nextSearch()' + */ + public void testSimple() { + tok = toSurtT("http://www.archive.org/foo"); + assertEquals("(org,archive,www,)/foo\t",tok.nextSearch()); + assertEquals("(org,archive,www,)/foo",tok.nextSearch()); + assertEquals("(org,archive,www,",tok.nextSearch()); + assertEquals("(org,archive,www",tok.nextSearch()); + assertEquals("(org,archive",tok.nextSearch()); + assertEquals("(org",tok.nextSearch()); + assertNull(tok.nextSearch()); + } + /** test */ + public void testSlashPath() { + tok = toSurtT("http://www.archive.org/"); + assertEquals("(org,archive,www,)/\t",tok.nextSearch()); + assertEquals("(org,archive,www,)/",tok.nextSearch()); + assertEquals("(org,archive,www,",tok.nextSearch()); + assertEquals("(org,archive,www",tok.nextSearch()); + assertEquals("(org,archive",tok.nextSearch()); + assertEquals("(org",tok.nextSearch()); + assertNull(tok.nextSearch()); + } + + /** test */ + public void testEmptyPath() { + tok = toSurtT("http://www.archive.org"); + assertEquals("(org,archive,www,)/\t",tok.nextSearch()); + assertEquals("(org,archive,www,)/",tok.nextSearch()); + assertEquals("(org,archive,www,",tok.nextSearch()); + assertEquals("(org,archive,www",tok.nextSearch()); + assertEquals("(org,archive",tok.nextSearch()); + assertEquals("(org",tok.nextSearch()); + assertNull(tok.nextSearch()); + } + + + /** test */ + public void testEmptyPathMore() { + tok = toSurtT("http://brad.www.archive.org"); + assertEquals("(org,archive,www,brad,)/\t",tok.nextSearch()); + assertEquals("(org,archive,www,brad,)/",tok.nextSearch()); + assertEquals("(org,archive,www,brad,",tok.nextSearch()); + assertEquals("(org,archive,www,brad",tok.nextSearch()); + assertEquals("(org,archive,www",tok.nextSearch()); + assertEquals("(org,archive",tok.nextSearch()); + assertEquals("(org",tok.nextSearch()); + assertNull(tok.nextSearch()); + } + /** test */ + public void testLongPathMore() { + tok = toSurtT("http://brad.www.archive.org/one/two"); + assertEquals("(org,archive,www,brad,)/one/two\t",tok.nextSearch()); + assertEquals("(org,archive,www,brad,)/one/two",tok.nextSearch()); + assertEquals("(org,archive,www,brad,)/one",tok.nextSearch()); + assertEquals("(org,archive,www,brad,",tok.nextSearch()); + assertEquals("(org,archive,www,brad",tok.nextSearch()); + assertEquals("(org,archive,www",tok.nextSearch()); + assertEquals("(org,archive",tok.nextSearch()); + assertEquals("(org",tok.nextSearch()); + assertNull(tok.nextSearch()); + } + /** test */ + public void testShortPathHash() { + tok = toSurtT("http://www.archive.org/one/two#hash"); + assertEquals("(org,archive,www,)/one/two\t",tok.nextSearch()); + assertEquals("(org,archive,www,)/one/two",tok.nextSearch()); + assertEquals("(org,archive,www,)/one",tok.nextSearch()); + assertEquals("(org,archive,www,",tok.nextSearch()); + assertEquals("(org,archive,www",tok.nextSearch()); + assertEquals("(org,archive",tok.nextSearch()); + assertEquals("(org",tok.nextSearch()); + assertNull(tok.nextSearch()); + } + /** test */ + public void testCGI1() { + tok = toSurtT("http://www.archive.org/cgi?foobar"); + assertEquals("(org,archive,www,)/cgi?foobar\t",tok.nextSearch()); + assertEquals("(org,archive,www,)/cgi?foobar",tok.nextSearch()); + assertEquals("(org,archive,www,)/cgi",tok.nextSearch()); + assertEquals("(org,archive,www,",tok.nextSearch()); + assertEquals("(org,archive,www",tok.nextSearch()); + assertEquals("(org,archive",tok.nextSearch()); + assertEquals("(org",tok.nextSearch()); + assertNull(tok.nextSearch()); + } + /** test */ + public void testPort() { + tok = toSurtT("http://www.archive.org:8080/cgi?foobar"); + assertEquals("(org,archive,www,:8080)/cgi?foobar\t",tok.nextSearch()); + assertEquals("(org,archive,www,:8080)/cgi?foobar",tok.nextSearch()); + assertEquals("(org,archive,www,:8080)/cgi",tok.nextSearch()); + assertEquals("(org,archive,www,:8080",tok.nextSearch()); + assertEquals("(org,archive,www",tok.nextSearch()); + assertEquals("(org,archive",tok.nextSearch()); + assertEquals("(org",tok.nextSearch()); + assertNull(tok.nextSearch()); + } + /** test */ + public void testLogin() { + tok = toSurtT("http://brad@www.archive.org/cgi?foobar"); + assertEquals("(org,archive,www,@brad)/cgi?foobar\t",tok.nextSearch()); + assertEquals("(org,archive,www,@brad)/cgi?foobar",tok.nextSearch()); + assertEquals("(org,archive,www,@brad)/cgi",tok.nextSearch()); + assertEquals("(org,archive,www,@brad",tok.nextSearch()); + assertEquals("(org,archive,www",tok.nextSearch()); + assertEquals("(org,archive",tok.nextSearch()); + assertEquals("(org",tok.nextSearch()); + assertNull(tok.nextSearch()); + } + /** test */ + public void testLoginPass() { + tok = toSurtT("http://brad:pass@www.archive.org/cgi?foobar"); + assertEquals("(org,archive,www,@brad:pass)/cgi?foobar\t",tok.nextSearch()); + assertEquals("(org,archive,www,@brad:pass)/cgi?foobar",tok.nextSearch()); + assertEquals("(org,archive,www,@brad:pass)/cgi",tok.nextSearch()); + assertEquals("(org,archive,www,@brad:pass",tok.nextSearch()); + assertEquals("(org,archive,www",tok.nextSearch()); + assertEquals("(org,archive",tok.nextSearch()); + assertEquals("(org",tok.nextSearch()); + assertNull(tok.nextSearch()); + } +// /** test */ + // leave this guy out for now: was a bug in Heritrix thus archive-commons + // wait for new jar... +// public void testLoginPassPort() { +// tok = toSurtT("http://brad:pass@www.archive.org:8080/cgi?foobar"); +// assertEquals("(org,archive,www,:8080@brad:pass)/cgi?foobar\t",tok.nextSearch()); +// assertEquals("(org,archive,www,:8080@brad:pass)/cgi?foobar",tok.nextSearch()); +// assertEquals("(org,archive,www,:8080@brad:pass)/cgi",tok.nextSearch()); +// assertEquals("(org,archive,www,:8080@brad:pass",tok.nextSearch()); +// assertEquals("(org,archive,www,:8080",tok.nextSearch()); +// assertEquals("(org,archive,www",tok.nextSearch()); +// assertEquals("(org,archive",tok.nextSearch()); +// assertEquals("(org",tok.nextSearch()); +// assertNull(tok.nextSearch()); +// } +// + + private SURTTokenizer toSurtT(final String u) { + SURTTokenizer tok = null; + try { + tok = new SURTTokenizer(u); + } catch (URIException e) { + e.printStackTrace(); + assertFalse("URL Exception " + e.getLocalizedMessage(),true); + } + return tok; + } + +} diff --git a/commons/src/test/java/org/archive/uid/UUIDGeneratorTest.java b/commons/src/test/java/org/archive/uid/UUIDGeneratorTest.java new file mode 100644 index 00000000..282f75b3 --- /dev/null +++ b/commons/src/test/java/org/archive/uid/UUIDGeneratorTest.java @@ -0,0 +1,48 @@ +/* $Id$ + * + * Created on July 27th, 2006 + * + * Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.uid; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.Map; + +import junit.framework.TestCase; + +/** + * @author stack + * @version $Revision$ $Date$ + */ +public class UUIDGeneratorTest extends TestCase { + public void testQualifyRecordID() throws URISyntaxException { + Generator g = new UUIDGenerator(); + URI uri = g.getRecordID(); + Map qualifiers = new HashMap(); + qualifiers.put("a", "b"); + URI nuURI = g.qualifyRecordID(uri, qualifiers); + assertNotSame(uri, nuURI); + qualifiers.put("c", "d"); + nuURI = g.qualifyRecordID(nuURI, qualifiers); + assertNotSame(uri, nuURI); + } +} diff --git a/commons/src/test/java/org/archive/util/ArchiveUtilsTest.java b/commons/src/test/java/org/archive/util/ArchiveUtilsTest.java new file mode 100644 index 00000000..79a21520 --- /dev/null +++ b/commons/src/test/java/org/archive/util/ArchiveUtilsTest.java @@ -0,0 +1,402 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.util; + +import java.text.ParseException; +import java.util.Date; +import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicInteger; + +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; + +/** + * JUnit test suite for ArchiveUtils + * + * @author James Casey + * @version $Id$ + */ +public class ArchiveUtilsTest extends TestCase { + + /** + * Create a new ArchiveUtilsTest object + * + * @param testName the name of the test + */ + public ArchiveUtilsTest(final String testName) { + super(testName); + } + + /** + * run all the tests for ArchiveUtilsTest + * + * @param argv the command line arguments + */ + public static void main(String argv[]) { + junit.textui.TestRunner.run(suite()); + } + + /** + * return the suite of tests for ArchiveUtilsTest + * + * @return the suite of test + */ + public static Test suite() { + return new TestSuite(ArchiveUtilsTest.class); + } + + /** check the getXXDigitDate() methods produce valid dates*/ + public void testGetXXDigitDate() { + // TODO - we only really test the date lengths here. How to test + // other stuff well ? + final String date12 = ArchiveUtils.get12DigitDate(); + assertEquals("12 digits", 12, date12.length()); + + final String date14 = ArchiveUtils.get14DigitDate(); + assertEquals("14 digits", 14, date14.length()); + + final String date17 = ArchiveUtils.get17DigitDate(); + assertEquals("17 digits", 17, date17.length()); + + // now parse, and check they're all within 1 minute + + try { + final long long12 = ArchiveUtils.parse12DigitDate(date12).getTime(); + long long14 = ArchiveUtils.parse14DigitDate(date14).getTime(); + long long17 = ArchiveUtils.parse17DigitDate(date17).getTime(); + + assertClose("12 and 14 close", long12, long14, 600000); + assertClose("12 and 17 close", long12, long17, 600000); + assertClose("14 and 17 close", long14, long17, 600000); + } catch (ParseException e) { + fail("Could not parse a date : " + e.getMessage()); + } + } + + /** check that getXXDigitDate(long) does the right thing */ + public void testGetXXDigitDateLong() { + final long now = System.currentTimeMillis(); + final String date12 = ArchiveUtils.get12DigitDate(now); + assertEquals("12 digits", 12, date12.length()); + + final String date14 = ArchiveUtils.get14DigitDate(now); + assertEquals("14 digits", 14, date14.length()); + assertEquals("first twelve digits same as date12", date12, date14.substring(0, 12)); + final String date17 = ArchiveUtils.get17DigitDate(now); + assertEquals("17 digits", 17, date17.length()); + assertEquals("first twelve digits same as date12", date12, date17.substring(0, 12)); + assertEquals("first fourteen digits same as date14", date14, date17.substring(0, 14)); + } + + /** + * Check that parseXXDigitDate() works + * + * @throws ParseException + */ + public void testParseXXDigitDate() throws ParseException { + // given a date, check it get resolved properly + // It's 02 Jan 2004, 12:40:02.111 + final String date = "20040102124002111"; + try { + final long long12 = ArchiveUtils.parse12DigitDate(date.substring(0, 12)).getTime(); + final long long14 = ArchiveUtils.parse14DigitDate(date.substring(0, 14)).getTime(); + final long long17 = ArchiveUtils.parse17DigitDate(date).getTime(); + + assertClose("12 and 14 close", long12, long14, 600000); + assertClose("12 and 17 close", long12, long17, 600000); + assertClose("14 and 17 close", long14, long17, 600000); + } catch (ParseException e) { + fail("Could not parse a date : " + e.getMessage()); + } + } + + public void testTooShortParseDigitDate() throws ParseException { + String d = "X"; + boolean b = false; + try { + ArchiveUtils.getDate(d); + } catch (ParseException e) { + b = true; + } + assertTrue(b); + + Date date = ArchiveUtils.getDate("1999"); + assertTrue(date.getTime() == 915148800000L); + + b = false; + try { + ArchiveUtils.getDate("19991"); + } catch (ParseException e) { + b = true; + } + assertTrue(b); + + ArchiveUtils.getDate("19990101"); + ArchiveUtils.getDate("1999010101"); + ArchiveUtils.getDate("19990101010101"); + ArchiveUtils.getDate("1960"); + } + + /** check that parse12DigitDate doesn't accept a bad date */ + public void testBad12Date() { + // now try a badly formed dates + assertBad12DigitDate("a-stringy-digit-date"); + assertBad12DigitDate("20031201"); // too short + } + + /** + * check that parse14DigitDate doesn't accept a bad date + */ + public void testBad14Date() { + // now try a badly formed dates + assertBad14DigitDate("a-stringy-digit-date"); + assertBad14DigitDate("20031201"); // too short + assertBad14DigitDate("200401021240"); // 12 digit + } + /** + * check that parse12DigitDate doesn't accept a bad date + */ + public void testBad17Date() { + // now try a badly formed dates + assertBad17DigitDate("a-stringy-digit-date"); + assertBad17DigitDate("20031201"); // too short + assertBad17DigitDate("200401021240"); // 12 digit + assertBad17DigitDate("20040102124002"); // 14 digit + } + + /** check that padTo(String) works */ + public void testPadToString() { + assertEquals("pad to one (smaller)", "foo", ArchiveUtils.padTo("foo", 1)); + assertEquals("pad to 0 (no sense)", "foo", ArchiveUtils.padTo("foo", 0)); + assertEquals("pad to neg (nonsense)", "foo", ArchiveUtils.padTo("foo", 0)); + assertEquals("pad to 4", " foo", ArchiveUtils.padTo("foo", 4)); + assertEquals("pad to 10", " foo", ArchiveUtils.padTo("foo", 10)); + } + + /** + * check that padTo(int) works + */ + public void testPadToInt() { + assertEquals("pad to one (smaller)", "123", ArchiveUtils.padTo(123, 1)); + assertEquals("pad to 0 (no sense)", "123", ArchiveUtils.padTo(123, 0)); + assertEquals("pad to neg (nonsense)", "123", ArchiveUtils.padTo(123, 0)); + assertEquals("pad to 4", " 123", ArchiveUtils.padTo(123, 4)); + assertEquals("pad to 10", " 123", ArchiveUtils.padTo(123, 10)); + assertEquals("pad -123 to 10", " -123", ArchiveUtils.padTo(-123, 10)); + } + + /** check that byteArrayEquals() works */ + public void testByteArrayEquals() { + // foo == foo2, foo != bar, foo != bar2 + byte[] foo = new byte[10], bar = new byte[20]; + byte[] foo2 = new byte[10], bar2 = new byte[10]; + + for (byte i = 0; i < 10 ; ++i) { + foo[i] = foo2[i] = bar[i] = i; + bar2[i] = (byte)(01 + i); + } + assertTrue("two nulls", ArchiveUtils.byteArrayEquals(null, null)); + assertFalse("lhs null", ArchiveUtils.byteArrayEquals(null, foo)); + assertFalse("rhs null", ArchiveUtils.byteArrayEquals(foo, null)); + + // now check with same length, with same (foo2) and different (bar2) + // contents + assertFalse("different lengths", ArchiveUtils.byteArrayEquals(foo, bar)); + + assertTrue("same to itself", ArchiveUtils.byteArrayEquals(foo, foo)); + assertTrue("same contents", ArchiveUtils.byteArrayEquals(foo, foo2)); + assertFalse("different contents", ArchiveUtils.byteArrayEquals(foo, bar2)); + } + + /** test doubleToString() */ + public void testDoubleToString(){ + double test = 12.345; + assertTrue( + "cecking zero precision", + ArchiveUtils.doubleToString(test, 0).equals("12")); + assertTrue( + "cecking 2 character precision", + ArchiveUtils.doubleToString(test, 2).equals("12.34")); + assertTrue( + "cecking precision higher then the double has", + ArchiveUtils.doubleToString(test, 65).equals("12.345")); + } + + + public void testFormatBytesForDisplayPrecise(){ + assertEquals("formating negative number", "0 B", ArchiveUtils + .formatBytesForDisplay(-1)); + assertEquals("0 bytes", "0 B", ArchiveUtils + .formatBytesForDisplay(0)); + assertEquals("1023 bytes", "1,023 B", ArchiveUtils + .formatBytesForDisplay(1023)); + assertEquals("1025 bytes", "1.0 KB", ArchiveUtils + .formatBytesForDisplay(1025)); + // expected display values taken from Google calculator + assertEquals("10,000 bytes", "9.8 KB", + ArchiveUtils.formatBytesForDisplay(10000)); + assertEquals("1,000,000 bytes", "977 KB", + ArchiveUtils.formatBytesForDisplay(1000000)); + assertEquals("100,000,000 bytes", "95 MB", + ArchiveUtils.formatBytesForDisplay(100000000)); + assertEquals("100,000,000,000 bytes", "93 GB", + ArchiveUtils.formatBytesForDisplay(100000000000L)); + assertEquals("100,000,000,000,000 bytes", "91 TB", + ArchiveUtils.formatBytesForDisplay(100000000000000L)); + assertEquals("100,000,000,000,000,000 bytes", "90,949 TB", + ArchiveUtils.formatBytesForDisplay(100000000000000000L)); + } + + /* + * helper methods + */ + + /** check that this is a bad date, and fail() if so. + * + * @param date the 12digit date to check + */ + private void assertBad12DigitDate(final String date) { + try { + ArchiveUtils.parse12DigitDate(date); + } catch (ParseException e) { + return; + } + fail("Expected exception on parse of : " + date); + + } + /** + * check that this is a bad date, and fail() if so. + * + * @param date the 14digit date to check + */ + private void assertBad14DigitDate(final String date) { + try { + ArchiveUtils.parse14DigitDate(date); + } catch (ParseException e) { + return; + } + fail("Expected exception on parse of : " + date); + + } + + /** + * check that this is a bad date, and fail() if so. + * + * @param date the 17digit date to check + */ + private void assertBad17DigitDate(final String date) { + try { + ArchiveUtils.parse17DigitDate(date); + } catch (ParseException e) { + return; + } + fail("Expected exception on parse of : " + date); + + } + + /** check that two longs are within a given delta */ + private void assertClose(String desc, long date1, long date2, long delta) { + assertTrue(desc, date1 == date2 || + (date1 < date2 && date2 < (date1 + delta)) || + (date2 < date1 && date1 < (date2 + delta))); + } + + public void testArrayToLong() { + testOneArrayToLong(-1); + testOneArrayToLong(1); + testOneArrayToLong(1000); + testOneArrayToLong(Integer.MAX_VALUE); + } + + private void testOneArrayToLong(final long testValue) { + byte [] a = new byte[8]; + ArchiveUtils.longIntoByteArray(testValue, a, 0); + final long l = ArchiveUtils.byteArrayIntoLong(a, 0); + assertEquals(testValue, l); + } + + public void testSecondsSinceEpochCalculation() throws ParseException { + assertEquals(ArchiveUtils.secondsSinceEpoch("20010909014640"), + "1000000000"); + assertEquals(ArchiveUtils.secondsSinceEpoch("20010909014639"), + "0999999999"); + assertEquals(ArchiveUtils.secondsSinceEpoch("19700101"), + "0000000000"); + assertEquals(ArchiveUtils.secondsSinceEpoch("2005"), "1104537600"); + assertEquals(ArchiveUtils.secondsSinceEpoch("200501"), "1104537600"); + assertEquals(ArchiveUtils.secondsSinceEpoch("20050101"), "1104537600"); + assertEquals(ArchiveUtils.secondsSinceEpoch("2005010100"), + "1104537600"); + boolean eThrown = false; + try { + ArchiveUtils.secondsSinceEpoch("20050"); + } catch (IllegalArgumentException e) { + eThrown = true; + } + assertTrue(eThrown); + } + + public static void testZeroPadInteger() { + assertEquals(ArchiveUtils.zeroPadInteger(1), "0000000001"); + assertEquals(ArchiveUtils.zeroPadInteger(1000000000), "1000000000"); + } + + /** + * Test stable behavior of date formatting under heavy concurrency. + * + * @throws InterruptedException + */ + public static void testDateFormatConcurrency() throws InterruptedException { + final int COUNT = 1000; + Thread [] ts = new Thread[COUNT]; + final Semaphore allDone = new Semaphore(-COUNT+1); + final AtomicInteger failures = new AtomicInteger(0); + for (int i = 0; i < COUNT; i++) { + Thread t = new Thread() { + public void run() { + long n = System.currentTimeMillis(); + final String d = ArchiveUtils.get17DigitDate(n); + for (int i = 0; i < 1000; i++) { + try { + sleep(10); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + String d2 = ArchiveUtils.get17DigitDate(n); + if(!d.equals(d2)) { + failures.incrementAndGet(); + break; + } + } + allDone.release(); + } + }; + ts[i] = t; + ts[i].setName(Integer.toString(i)); + ts[i].start(); + while(!ts[i].isAlive()) /* Wait for thread to spin up*/; + } + allDone.acquire(); // wait for all threads to finish + assertEquals(failures.get()+" format mismatches",0,failures.get()); + } +} + diff --git a/commons/src/test/java/org/archive/util/CachedBdbMapTest.java b/commons/src/test/java/org/archive/util/CachedBdbMapTest.java new file mode 100644 index 00000000..6d602829 --- /dev/null +++ b/commons/src/test/java/org/archive/util/CachedBdbMapTest.java @@ -0,0 +1,89 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.util; + +import java.io.File; +import java.util.HashMap; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.io.FileUtils; + +/** + * @author stack + * @version $Date$, $Revision$ + */ +public class CachedBdbMapTest extends TmpDirTestCase { + File envDir; + private CachedBdbMap> cache; + + @SuppressWarnings("unchecked") + protected void setUp() throws Exception { + super.setUp(); + this.envDir = new File(getTmpDir(),"CachedBdbMapTest"); + this.envDir.mkdirs(); + this.cache = new CachedBdbMap(this.envDir, + this.getClass().getName(), String.class, HashMap.class); + } + + protected void tearDown() throws Exception { + this.cache.close(); + FileUtils.deleteDirectory(this.envDir); + super.tearDown(); + } + + public void testBackingDbGetsUpdated() { + // Enable all logging. Up the level on the handlers and then + // on the big map itself. + Handler [] handlers = Logger.getLogger("").getHandlers(); + for (int index = 0; index < handlers.length; index++) { + handlers[index].setLevel(Level.FINEST); + } + Logger.getLogger(CachedBdbMap.class.getName()). + setLevel(Level.FINEST); + // Set up values. + final String value = "value"; + final String key = "key"; + final int upperbound = 3; + // First put in empty hashmap. + for (int i = 0; i < upperbound; i++) { + this.cache.put(key + Integer.toString(i), new HashMap()); + } + // Now add value to hash map. + for (int i = 0; i < upperbound; i++) { + HashMap m = this.cache.get(key + Integer.toString(i)); + m.put(key, value); + } + this.cache.sync(); + for (int i = 0; i < upperbound; i++) { + HashMap m = this.cache.get(key + Integer.toString(i)); + String v = m.get(key); + if (v == null || !v.equals(value)) { + Logger.getLogger(CachedBdbMap.class.getName()). + warning("Wrong value " + i); + } + } + } + + public static void main(String [] args) { + junit.textui.TestRunner.run(CachedBdbMapTest.class); + } +} diff --git a/commons/src/test/java/org/archive/util/FileUtilsTest.java b/commons/src/test/java/org/archive/util/FileUtilsTest.java new file mode 100644 index 00000000..be815efd --- /dev/null +++ b/commons/src/test/java/org/archive/util/FileUtilsTest.java @@ -0,0 +1,282 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.util; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.math.LongRange; + + +/** + * FileUtils tests. + * + * @contributor stack + * @contributor gojomo + * @version $Date$, $Revision$ + */ +public class FileUtilsTest extends TmpDirTestCase { + private String srcDirName = FileUtilsTest.class.getName() + ".srcdir"; + private File srcDirFile = null; + private String tgtDirName = FileUtilsTest.class.getName() + ".tgtdir"; + private File tgtDirFile = null; + + protected File zeroLengthLinesUnix; + protected File zeroLengthLinesWindows; + + protected File smallLinesUnix; + protected File smallLinesWindows; + protected File largeLinesUnix; + protected File largeLinesWindows; + protected File nakedLastLineUnix; + protected File nakedLastLineWindows; + + + protected void setUp() throws Exception { + super.setUp(); + this.srcDirFile = new File(getTmpDir(), srcDirName); + this.srcDirFile.mkdirs(); + this.tgtDirFile = new File(getTmpDir(), tgtDirName); + this.tgtDirFile.mkdirs(); + addFiles(); + + zeroLengthLinesUnix = setUpLinesFile("zeroLengthLinesUnix",0,0,400,IOUtils.LINE_SEPARATOR_UNIX); + zeroLengthLinesWindows = setUpLinesFile("zeroLengthLinesUnix",0,0,400,IOUtils.LINE_SEPARATOR_WINDOWS); + + smallLinesUnix = setUpLinesFile("smallLinesUnix", 0, 25, 400, IOUtils.LINE_SEPARATOR_UNIX); + smallLinesWindows = setUpLinesFile("smallLinesWindows", 0, 25, 400, IOUtils.LINE_SEPARATOR_WINDOWS); + largeLinesUnix = setUpLinesFile("largeLinesUnix", 128, 256, 5, IOUtils.LINE_SEPARATOR_UNIX); + largeLinesWindows = setUpLinesFile("largeLinesWindows", 128, 256, 4096, IOUtils.LINE_SEPARATOR_WINDOWS); + + nakedLastLineUnix = setUpLinesFile("nakedLastLineUnix", 0, 50, 401, IOUtils.LINE_SEPARATOR_UNIX); + org.apache.commons.io.FileUtils.writeStringToFile(nakedLastLineUnix,"a"); + nakedLastLineWindows = setUpLinesFile("nakedLastLineWindows", 0, 50, 401, IOUtils.LINE_SEPARATOR_WINDOWS); + org.apache.commons.io.FileUtils.writeStringToFile(nakedLastLineWindows,"a"); + } + + private void addFiles() throws IOException { + addFiles(3, this.getName()); + } + + private void addFiles(final int howMany, final String baseName) + throws IOException { + for (int i = 0; i < howMany; i++) { + File.createTempFile(baseName, null, this.srcDirFile); + } + } + + private File setUpLinesFile(String name, int minLineSize, int maxLineSize, int lineCount, String lineEnding) throws IOException { + List lines = new LinkedList(); + StringBuilder sb = new StringBuilder(maxLineSize); + for(int i = 0; i lines = org.apache.commons.io.FileUtils.readLines(file); + verifyTailLines(file, lines, 1, 80); + verifyTailLines(file, lines, 5, 80); + verifyTailLines(file, lines, 10, 80); + verifyTailLines(file, lines, 20, 80); + verifyTailLines(file, lines, 100, 80); + verifyTailLines(file, lines, 1, 1); + verifyTailLines(file, lines, 5, 1); + verifyTailLines(file, lines, 10, 1); + verifyTailLines(file, lines, 20, 1); + verifyTailLines(file, lines, 100, 1); + } + + + private void verifyTailLines(File file, List lines, int count, int estimate) throws IOException { + List testLines; + testLines = getTestTailLines(file,count,estimate); + assertEquals("line counts not equal:"+file.getName()+" "+count+" "+estimate,lines.size(),testLines.size()); + assertEquals("lines not equal: "+file.getName()+" "+count+" "+estimate,lines,testLines); + } + + private List getTestTailLines(File file, int count, int estimate) throws IOException { + long pos = -1; + List testLines = new LinkedList(); + do { + List returnedLines = new LinkedList(); + LongRange range = FileUtils.pagedLines(file,pos,-count,returnedLines,estimate); + Collections.reverse(returnedLines); + testLines.addAll(returnedLines); + pos = range.getMinimumLong()-1; + } while (pos>=0); + Collections.reverse(testLines); + return testLines; + } + + public void testHeadLinesZeroLengthUnix() throws IOException { + verifyHeadLines(zeroLengthLinesUnix); + } + + public void testHeadLinesZeroLengthWindows() throws IOException { + verifyHeadLines(zeroLengthLinesWindows); + } + + public void testHeadLinesSmallUnix() throws IOException { + verifyHeadLines(smallLinesUnix); + } + + public void testHeadLinesLargeUnix() throws IOException { + verifyHeadLines(largeLinesUnix); + } + + public void testHeadLinesSmallWindows() throws IOException { + verifyHeadLines(smallLinesWindows); + } + + public void testHeadLinesLargeWindows() throws IOException { + verifyHeadLines(largeLinesWindows); + } + + public void testHeadLinesNakedUnix() throws IOException { + verifyHeadLines(nakedLastLineUnix); + } + + public void testHeadLinesNakedWindows() throws IOException { + verifyHeadLines(nakedLastLineWindows); + } + + + @SuppressWarnings("unchecked") + private void verifyHeadLines(File file) throws IOException { + List lines = org.apache.commons.io.FileUtils.readLines(file); + verifyHeadLines(file, lines, 1, 80); + verifyHeadLines(file, lines, 5, 80); + verifyHeadLines(file, lines, 10, 80); + verifyHeadLines(file, lines, 20, 80); + verifyHeadLines(file, lines, 100, 80); + verifyHeadLines(file, lines, 1, 1); + verifyHeadLines(file, lines, 5, 1); + verifyHeadLines(file, lines, 10, 1); + verifyHeadLines(file, lines, 20, 1); + verifyHeadLines(file, lines, 100, 1); + } + + + private void verifyHeadLines(File file, List lines, int count, int estimate) throws IOException { + List testLines; + testLines = getTestHeadLines(file,count,estimate); + assertEquals("line counts not equal:"+file.getName()+" "+count+" "+estimate,lines.size(),testLines.size()); + assertEquals("lines not equal: "+file.getName()+" "+count+" "+estimate,lines,testLines); + } + + private List getTestHeadLines(File file, int count, int estimate) throws IOException { + long pos = 0; + List testLines = new LinkedList(); + do { + LongRange range = FileUtils.pagedLines(file,pos,count,testLines,estimate); + pos = range.getMaximumLong(); + } while (posJames Casey + * @version $Id$ + */ +public class PaddingStringBufferTest extends TestCase { + /** + * Create a new PaddingStringBufferTest object + * + * @param testName the name of the test + */ + public PaddingStringBufferTest(final String testName) { + super(testName); + } + + /** + * run all the tests for PaddingStringBufferTest + * + * @param argv the command line arguments + */ + public static void main(String argv[]) { + junit.textui.TestRunner.run(suite()); + } + + /** + * return the suite of tests for PaddingStringBufferTest + * + * @return the suite of test + */ + public static Test suite() { + return new TestSuite(PaddingStringBufferTest.class); + } + + public void setUp() { + buf = new PaddingStringBuffer(); + } + + /** first check that padTo works ok, since all depends on it */ + public void testPadTo() { + PaddingStringBuffer retBuf; + assertEquals("nothing in buffer", "", buf.toString()); + retBuf = buf.padTo(5); + assertEquals("retBuf same as buf", retBuf, buf); + assertEquals("5 spaces", " ", buf.toString()); + + // now do a smaller value - nothing should happen + buf.padTo(4); + assertEquals("5 spaces", " ", buf.toString()); + + // now pad tro a greater length + buf.padTo(10); + assertEquals("10 spaces", " ", buf.toString()); + } + + /** test that append(String) works correctly */ + public void testAppendString() { + // a buf to hold the return buffer + PaddingStringBuffer retBuf; + assertEquals("nothing in buffer", "", buf.toString()); + retBuf = buf.append("foo"); + assertEquals("foo in buffer", "foo", buf.toString()); + assertEquals("retBuf good", retBuf.toString(), buf.toString()); + retBuf = buf.append("bar"); + assertEquals("foobar in buffer", "foobar", buf.toString()); + assertEquals("retBuf good", retBuf.toString(), buf.toString()); + } + + /** check the reset method clears the buffer */ + public void testReset() { + // append something into the buffer + assertEquals("nothing in buffer", "", buf.toString()); + buf.append("foo"); + assertEquals("buffer is 'foo'", "foo", buf.toString()); + buf.reset(); + assertEquals("nothing in buffer after reset", "", buf.toString()); + } + + /** test the raAppend(String) works in the simple cases */ + public void testRaAppend() { + // a buf to hold the return buffer + PaddingStringBuffer retBuf; + assertEquals("nothing in buffer", "", buf.toString()); + retBuf = buf.raAppend(5, "foo"); + assertEquals("foo in buffer", " foo", buf.toString()); + assertEquals("retBuf good", retBuf.toString(), buf.toString()); + retBuf = buf.raAppend(9, "bar"); + assertEquals("foobar in buffer", " foo bar", buf.toString()); + assertEquals("retBuf good", retBuf.toString(), buf.toString()); + + // now check with out-of-range columns - should just append + buf = new PaddingStringBuffer(); + buf.raAppend(-1, "foo"); + assertEquals("no padding for -1", "foo", buf.toString()); + buf = new PaddingStringBuffer(); + buf.raAppend(0, "foo"); + assertEquals("no padding for 0", "foo", buf.toString()); + + } + + /** test the newline() */ + public void testNewline(){ + assertEquals("nothing should be in the buffer", "", buf.toString()); + buf.newline(); + assertTrue("should contain newline", buf.toString().indexOf('\n')!=-1); + assertEquals("line position should be 0",0,buf.linePos); + } + + /** check what happens when we right append, but the string is longer + * than the space */ + public void testRaAppendWithTooLongString() { + buf.raAppend(3,"foobar"); + assertEquals("no padding when padding col less than string length", + "foobar", buf.toString()); + buf.reset(); + } + + /** check it all works with the length == the length of the string */ + public void testRaAppendWithExactLengthString() { + buf.raAppend(6, "foobar"); + buf.raAppend(12, "foobar"); + assertEquals("no padding with exact length string", + "foobarfoobar", buf.toString()); + } + + /** check that append(int) works */ + public void testAppendInt() { + buf.append((int)1); + assertEquals("buffer is '1'", "1", buf.toString()); + buf.append((int)234); + assertEquals("buffer is '1234'", "1234", buf.toString()); + } + + /** check that raAppend(int) works */ + public void testRaAppendInt() { + // right-append '1' to column 5 + buf.raAppend(5, (int)1); + assertEquals("buf is ' 1'", " 1", buf.toString()); + // try appending a too-long int + + buf.raAppend(6,(int)123); + assertEquals("'123' appended", " 1123", buf.toString()); + } + + /** check that append(long) works */ + public void testAppendLong() { + buf.append((long)1); + assertEquals("buffer is '1'", "1", buf.toString()); + buf.append((long)234); + assertEquals("buffer is '1234'", "1234", buf.toString()); + } + + /** check that raAppend(long) works */ + public void testRaAppendLong() { + // right-append '1' to column 5 + buf.raAppend(5, (long) 1); + assertEquals("buf is ' 1'", " 1", buf.toString()); + // try appending a too-long int + + buf.raAppend(6, (long) 123); + assertEquals("'123' appended", " 1123", buf.toString()); + } + + /** a temp buffer for testing with */ + private PaddingStringBuffer buf; +} + diff --git a/commons/src/test/java/org/archive/util/SURTTest.java b/commons/src/test/java/org/archive/util/SURTTest.java new file mode 100644 index 00000000..3a3c31f3 --- /dev/null +++ b/commons/src/test/java/org/archive/util/SURTTest.java @@ -0,0 +1,114 @@ +/* SURTTest + * + * $Id$ + * + * Created Tue Jan 20 14:17:59 PST 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +package org.archive.util; + +import org.apache.commons.httpclient.URIException; + +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; + +/** + * JUnit test suite for SURT + * + * @author gojomo + * @version $ Id$ + */ +public class SURTTest extends TestCase { + /** + * Create a new MemQueueTest object + * + * @param testName + * the name of the test + */ + public SURTTest(final String testName) { + super(testName); + } + + /** + * run all the tests for MemQueueTest + * + * @param argv + * the command line arguments + */ + public static void main(String argv[]) { + junit.textui.TestRunner.run(suite()); + } + + /** + * return the suite of tests for MemQueueTest + * + * @return the suite of test + */ + public static Test suite() { + return new TestSuite(SURTTest.class); + } + + public void testMisc() throws URIException { + assertEquals("", + "http://(org,archive,www,)", + SURT.fromURI("http://www.archive.org")); + + assertEquals("", + "http://(org,archive,www,)/movies/movies.php", + SURT.fromURI("http://www.archive.org/movies/movies.php")); + + assertEquals("", + "http://(org,archive,www,:8080)/movies/movies.php", + SURT.fromURI("http://www.archive.org:8080/movies/movies.php")); + + assertEquals("", + "http://(org,archive,www,@user:pass)/movies/movies.php", + SURT.fromURI("http://user:pass@www.archive.org/movies/movies.php")); + + assertEquals("", + "http://(org,archive,www,:8080@user:pass)/movies/movies.php", + SURT.fromURI("http://user:pass@www.archive.org:8080/movies/movies.php")); + + assertEquals("", + "http://(org,archive,www,)/movies/movies.php#top", + SURT.fromURI("http://www.archive.org/movies/movies.php#top")); + } + + public void testAtSymbolInPath() throws URIException { + assertEquals("@ in path", + "http://(com,example,www,)/foo@bar", + SURT.fromURI("http://www.example.com/foo@bar")); + } + + /** + * Verify that dotted-quad numeric IP address is unreversed as per change + * requested in: [ 1572391 ] SURTs for IP-address URIs unhelpful + * + * @throws URIException + */ + public void testDottedQuadAuthority() throws URIException { + assertEquals("dotted-quad IP authority", + "http://(127.2.34.5)/foo", + SURT.fromURI("http://127.2.34.5/foo")); + } +} + diff --git a/commons/src/test/java/org/archive/util/SurtPrefixSetTest.java b/commons/src/test/java/org/archive/util/SurtPrefixSetTest.java new file mode 100644 index 00000000..a71a4d28 --- /dev/null +++ b/commons/src/test/java/org/archive/util/SurtPrefixSetTest.java @@ -0,0 +1,162 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.util; + +import java.io.IOException; +import java.io.StringReader; +import java.util.Iterator; + +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; + +/** + * @author gojomo + */ +public class SurtPrefixSetTest extends TestCase { + private static final String ARCHIVE_ORG_DOMAIN_SURT = "http://(org,archive,"; + private static final String WWW_EXAMPLE_ORG_HOST_SURT = "http://(org,example,www,)"; + private static final String HOME_EXAMPLE_ORG_PATH_SURT = "http://(org,example,home,)/pages/"; + private static final String BOK_IS_REDUNDANT_SURT = "http://(is,bok,"; + private static final String IS_DOMAIN_SURT = "http://(is,"; + private static final String WWW_BOK_IS_REDUNDANT_SURT = "http://(is,bok,www"; + + private static final String TEST_SURT_LIST = + "# a test set of surt prefixes \n" + + ARCHIVE_ORG_DOMAIN_SURT + "\n" + + WWW_EXAMPLE_ORG_HOST_SURT + "\n" + + HOME_EXAMPLE_ORG_PATH_SURT + "\n" + + BOK_IS_REDUNDANT_SURT + " # is redundant\n" + + IS_DOMAIN_SURT + "\n" + + WWW_BOK_IS_REDUNDANT_SURT + " # is redundant\n"; + + /** + * Create a new SurtPrefixSetTest object + * + * @param testName + * the name of the test + */ + public SurtPrefixSetTest(final String testName) { + super(testName); + } + + /** + * run all the tests for SurtPrefixSetTest + * + * @param argv + * the command line arguments + */ + public static void main(String argv[]) { + junit.textui.TestRunner.run(suite()); + } + + /** + * return the suite of tests for SurtPrefixSetTest + * + * @return the suite of test + */ + public static Test suite() { + return new TestSuite(SurtPrefixSetTest.class); + } + + + + public void testMisc() throws IOException { + SurtPrefixSet surts = new SurtPrefixSet(); + StringReader sr = new StringReader(TEST_SURT_LIST); + surts.importFrom(sr); + + assertContains(surts,ARCHIVE_ORG_DOMAIN_SURT); + assertContains(surts,WWW_EXAMPLE_ORG_HOST_SURT); + assertContains(surts,HOME_EXAMPLE_ORG_PATH_SURT); + assertContains(surts,IS_DOMAIN_SURT); + + assertDoesntContain(surts,BOK_IS_REDUNDANT_SURT); + assertDoesntContain(surts,WWW_BOK_IS_REDUNDANT_SURT); + + assertContainsPrefix(surts,SURT.fromURI("http://example.is/foo")); + assertDoesntContainPrefix(surts,SURT.fromURI("http://home.example.org/foo")); + } + + /** + * @param surts + * @param string + */ + private void assertDoesntContainPrefix(SurtPrefixSet surts, String s) { + assertEquals(s+" is prefixed", surts.containsPrefixOf(s), false); + } + + /** + * @param surts + * @param string + */ + private void assertContainsPrefix(SurtPrefixSet surts, String s) { + assertEquals(s+" isn't prefixed", surts.containsPrefixOf(s), true); + } + + /** + * @param surts + * @param www_bok_is_redundant_surt2 + */ + private void assertDoesntContain(SurtPrefixSet surts, String s) { + assertEquals(s+" is present", surts.contains(s), false); + } + + /** + * @param archive_org_domain_surt2 + */ + private void assertContains(SurtPrefixSet surts, String s) { + assertEquals(s+" is missing", surts.contains(s), true); + } + + public void testImportFromUris() throws IOException { + String seed = "http://www.archive.org/index.html"; + assertEquals("Convert failed " + seed, + "http://(org,archive,www,)/", + makeSurtPrefix(seed)); + seed = "http://timmknibbs4senate.blogspot.com/"; + assertEquals("Convert failed " + seed, + "http://(com,blogspot,timmknibbs4senate,)/", + makeSurtPrefix(seed)); + seed = "https://one.two.three"; + assertEquals("Convert failed " + seed, + "http://(three,two,one,", + makeSurtPrefix(seed)); + seed = "https://xone.two.three/a/b/c/"; + assertEquals("Convert failed " + seed, + "http://(three,two,xone,)/a/b/c/", + makeSurtPrefix(seed)); + seed = "https://yone.two.three/a/b/c"; + assertEquals("Convert failed " + seed, + "http://(three,two,yone,)/a/b/", + makeSurtPrefix(seed)); + } + + private String makeSurtPrefix(String seed) { + SurtPrefixSet surts = new SurtPrefixSet(); + StringReader sr = new StringReader(seed); + surts.importFromUris(sr); + String result = null; + for (Iterator i = surts.iterator(); i.hasNext();) { + result = (String)i.next(); + } + return result; + } +} diff --git a/commons/src/test/java/org/archive/util/TextUtilsTest.java b/commons/src/test/java/org/archive/util/TextUtilsTest.java new file mode 100644 index 00000000..e762091b --- /dev/null +++ b/commons/src/test/java/org/archive/util/TextUtilsTest.java @@ -0,0 +1,130 @@ +/* TextUtilsTest.java + * + * $Id$ + * + * Created Tue Jan 20 14:17:59 PST 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +package org.archive.util; + +import java.util.regex.Matcher; + +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; + +/** + * JUnit test suite for TextUtils + * + * @author gojomo + * @version $ Id$ + */ +public class TextUtilsTest extends TestCase { + /** + * Create a new TextUtilsTest object + * + * @param testName + * the name of the test + */ + public TextUtilsTest(final String testName) { + super(testName); + } + + /** + * run all the tests for TextUtilsTest + * + * @param argv + * the command line arguments + */ + public static void main(String argv[]) { + junit.textui.TestRunner.run(suite()); + } + + /** + * return the suite of tests for MemQueueTest + * + * @return the suite of test + */ + public static Test suite() { + return new TestSuite(TextUtilsTest.class); + } + + public void testMatcherRecycling() { + String pattern = "f.*"; + Matcher m1 = TextUtils.getMatcher(pattern,"foo"); + assertTrue("matcher against 'foo' problem", m1.matches()); + TextUtils.recycleMatcher(m1); + Matcher m2 = TextUtils.getMatcher(pattern,""); + assertFalse("matcher against '' problem", m2.matches()); + assertTrue("matcher not recycled",m1==m2); + // now verify proper behavior without recycling + Matcher m3 = TextUtils.getMatcher(pattern,"fuggedaboutit"); + assertTrue("matcher against 'fuggedaboutit' problem",m3.matches()); + assertFalse("matcher was recycled",m3==m2); + } + + public void testGetFirstWord() { + final String firstWord = "one"; + String tmpStr = TextUtils.getFirstWord(firstWord + " two three"); + assertTrue("Failed to get first word 1 " + tmpStr, + tmpStr.equals(firstWord)); + tmpStr = TextUtils.getFirstWord(firstWord); + assertTrue("Failed to get first word 2 " + tmpStr, + tmpStr.equals(firstWord)); + } + + public void testUnescapeHtml() { + final String abc = "abc"; + CharSequence cs = TextUtils.unescapeHtml("abc"); + assertEquals(cs, abc); + final String backwards = "aaa;lt&aaa"; + cs = TextUtils.unescapeHtml(backwards); + assertEquals(cs, backwards); + final String ampersand = "aaa&aaa"; + cs = TextUtils.unescapeHtml(ampersand); + assertEquals(cs, ampersand); + final String encodedAmpersand = "aaa&aaa"; + cs = TextUtils.unescapeHtml(encodedAmpersand); + assertEquals(cs, ampersand); + final String encodedQuote = "aaa'aaa"; + cs = TextUtils.unescapeHtml(encodedQuote); + assertEquals(cs, "aaa'aaa"); + final String entityQuote = "aaa"aaa"; + cs = TextUtils.unescapeHtml(entityQuote); + assertEquals(cs, "aaa\"aaa"); + final String hexencoded = "aaa aaa"; + cs = TextUtils.unescapeHtml(hexencoded); + assertEquals(cs, "aaa\naaa"); + final String zeroPos = "&aaa"; + cs = TextUtils.unescapeHtml(zeroPos); + assertEquals(cs, "&aaa"); + } + + public void testUnescapeHtmlWithDanglingAmpersand() { + final String mixedEncodedAmpersand1 = "aaa&aaa&aaa"; + CharSequence cs = TextUtils.unescapeHtml(mixedEncodedAmpersand1); + assertEquals("aaa&aaa&aaa",cs); + final String mixedEncodedAmpersand2 = "aaa&aaa&aaa&aaa"; + cs = TextUtils.unescapeHtml(mixedEncodedAmpersand2); + assertEquals("aaa&aaa&aaa&aaa",cs); + } +} + diff --git a/commons/src/test/java/org/archive/util/TransformTest.java b/commons/src/test/java/org/archive/util/TransformTest.java new file mode 100644 index 00000000..51641fcf --- /dev/null +++ b/commons/src/test/java/org/archive/util/TransformTest.java @@ -0,0 +1,126 @@ +package org.archive.util; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; + +import junit.framework.TestCase; + +/** + * Tests the {@link dex.misc.Transform} class. + */ +public class TransformTest extends TestCase { + + // Convert integers to strings, eliminating negative numbers + private static class PositiveToString + implements Transformer { + public String transform(Integer i) { + if (i < 0) { + return null; + } + return i.toString(); + } + } + + /** + * Tests using a simple Transformer. The Transformer changes + * positive integers into strings. The test sets up a + * list of random integers, remembering which ones are + * positive. The Transform is created, and the Transform's + * contents are compared against the list of remembered positive + * integers. + */ + public void testTransform() { + Transformer transformer = new PositiveToString(); + + // Transform of an empty collection should be empty. + List empty = new ArrayList(); + assertTrue(new Transform(empty, transformer).isEmpty()); + + // Some simple test data. + Integer[] testData = new Integer[] { -5, 3, 2, -11, 0, 111, -161 }; + String[] expected = new String[] { "3", "2", "0", "111" }; + List list = Arrays.asList(testData); + Transform c = new Transform(list, + transformer); + List expectedList = Arrays.asList(expected); + assertEquals(new ArrayList(c), expectedList); + + // Same test as above, with random data + for (int i = 0; i < 100; i++) { + randomTest(); + } + } + + private void randomTest() { + Transformer transformer = new PositiveToString(); + Random random = new Random(); + int max = random.nextInt(1024) + 10; + List testData = new ArrayList(max); + List expected = new ArrayList(max); + for (int i = 0; i < max; i++) { + int e = random.nextInt(); + testData.add(e); + if (e >= 0) { + expected.add(Integer.toString(e)); + } + } + + Transform c = new Transform(testData, + transformer); + List results = new ArrayList(c); + assertEquals(expected, results); + } + + /** + * Tests the static subclasses method. The test sets up a list of + * Number instances that may contain random Double, Float, Integer + * or Long values. The Long values are remembered. The subclasses + * method is used to create a Transform containing only the Long + * values. The Transform is compared against the list of remembered + * Long values. + */ + public void testSubclasses() { + Random random = new Random(); + for (int i = 0; i < 100; i++) { + int max = random.nextInt(1024) + 10; + List testData = new ArrayList(max); + List expected = new ArrayList(max); + for (int j = 0; j < max; j++) { + int v = random.nextInt(4); + switch (v) { + case 0: + long l = random.nextLong(); + testData.add(l); + expected.add(l); + break; + case 1: + testData.add(random.nextInt()); + break; + case 2: + testData.add(random.nextDouble()); + break; + case 3: + testData.add(random.nextFloat()); + break; + } + } + Collection c = Transform.subclasses(testData, Long.class); + List results = new ArrayList(c); + assertEquals(expected, results); + } + } + + public void testSingleton() { + Set set = new HashSet(); + set.add(3); + Collection c = Transform.subclasses(set, Integer.class); + for (Integer i : c) { + System.out.println(i); + } + } +} diff --git a/commons/src/test/java/org/archive/util/anvl/ANVLRecordTest.java b/commons/src/test/java/org/archive/util/anvl/ANVLRecordTest.java new file mode 100644 index 00000000..80d7d18d --- /dev/null +++ b/commons/src/test/java/org/archive/util/anvl/ANVLRecordTest.java @@ -0,0 +1,125 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.util.anvl; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.util.Map; + +import junit.framework.TestCase; + +public class ANVLRecordTest extends TestCase { + public void testAdd() throws Exception { + ANVLRecord am = new ANVLRecord(); + am.add(new Element(new Label("entry"))); + am.add(new Element(new Label("who"), + new Value("Gilbert, W.S. | Sullivan, Arthur"))); + am.add(new Element(new Label("what"), + new Value("\rThe Yeoman of \rthe guard"))); + am.add(new Element(new Label("what"), + new Value("The Yeoman of\r\n the guard"))); + am.add(new Element(new Label("what"), + new Value("The Yeoman of \n\tthe guard"))); + am.add(new Element(new Label("what"), + new Value("The Yeoman of \r the guard"))); + am.add(new Element(new Label("when/created"), + new Value("1888"))); + System.out.println(am.toString()); + Map m = am.asMap(); + System.out.println(m.toString()); + } + + public void testEmptyRecord() throws Exception { + byte [] b = ANVLRecord.EMPTY_ANVL_RECORD.getUTF8Bytes(); + assertEquals(b.length, 2); + assertEquals(b[0], '\r'); + assertEquals(b[1], '\n'); + } + + public void testFolding() throws Exception { + ANVLRecord am = new ANVLRecord(); + Exception e = null; + try { + am.addLabel("Label with \n in it"); + } catch (IllegalArgumentException iae) { + e = iae; + } + assertTrue(e != null && e instanceof IllegalArgumentException); + am.addLabelValue("label", "value with \n in it"); + } + + public void testParse() throws UnsupportedEncodingException, IOException { + String record = " a: b\r\n#c#\r\nc:d\r\n \t\t\r\t\n\te" + + "\r\nx:\r\n # z\r\n\r\n"; + ANVLRecord r = ANVLRecord.load(new ByteArrayInputStream( + record.getBytes("ISO-8859-1"))); + System.out.println(r); + assertEquals(r.get(0).toString(), "a: b"); + record = " a: b\r\n\r\nsdfsdsdfds"; + r = ANVLRecord.load(new ByteArrayInputStream( + record.getBytes("ISO-8859-1"))); + System.out.println(r); + record = "x:\r\n # z\r\ny:\r\n\r\n"; + r = ANVLRecord.load(new ByteArrayInputStream( + record.getBytes("ISO-8859-1"))); + System.out.println(r); + assertEquals(r.get(0).toString(), "x:"); + } + + public void testExampleParse() + throws UnsupportedEncodingException, IOException { + final String sample = "entry:\t\t\r\n# first ###draft\r\n" + + "who:\tGilbert, W.S. | Sullivan, Arthur\r\n" + + "what:\tThe Yeoman of\r\n" + + "\t\tthe Guard\r\n" + + "when/created:\t 1888\r\n\r\n"; + ANVLRecord r = ANVLRecord.load(new ByteArrayInputStream( + sample.getBytes("ISO-8859-1"))); + System.out.println(r); + } + + public void testPoundLabel() + throws UnsupportedEncodingException, IOException { + final String sample = "ent#ry:\t\t\r\n# first ###draft\r\n" + + "who:\tGilbert, W.S. | Sullivan, Arthur\r\n" + + "what:\tThe Yeoman of\r\n" + + "\t\tthe Guard\r\n" + + "when/created:\t 1888\r\n\r\n"; + ANVLRecord r = ANVLRecord.load(sample); + System.out.println(r); + } + + public void testNewlineLabel() + throws UnsupportedEncodingException, IOException { + final String sample = "ent\nry:\t\t\r\n# first ###draft\r\n" + + "who:\tGilbert, W.S. | Sullivan, Arthur\r\n" + + "what:\tThe Yeoman of\r\n" + + "\t\tthe Guard\r\n" + + "when/created:\t 1888\r\n\r\n"; + IllegalArgumentException iae = null; + try { + ANVLRecord.load(sample); + } catch(IllegalArgumentException e) { + iae = e; + } + assertTrue(iae != null); + } +} diff --git a/commons/src/test/java/org/archive/util/fingerprint/ArrayLongFPCacheTest.java b/commons/src/test/java/org/archive/util/fingerprint/ArrayLongFPCacheTest.java new file mode 100644 index 00000000..761fd710 --- /dev/null +++ b/commons/src/test/java/org/archive/util/fingerprint/ArrayLongFPCacheTest.java @@ -0,0 +1,86 @@ +/* ArrayLongFPCacheTest +* +* $Id$ +* +* Created on Oct 5, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util.fingerprint; + +import junit.framework.TestCase; + +/** + * Unit tests for ArrayLongFPCache. + * + * @author gojomo + */ +public class ArrayLongFPCacheTest extends TestCase { + + public void testAdd() { + long testVal = 123456L; + ArrayLongFPCache cache = new ArrayLongFPCache(); + assertFalse("contains test value pre-add",cache.contains(testVal)); + assertFalse("contains test value pre-add",cache.contains(-testVal)); + cache.add(testVal); + cache.add(-testVal); + assertTrue("should contain after add",cache.contains(testVal)); + assertTrue("should contain after add",cache.contains(-testVal)); + } + + public void testContains() { + long testVal1 = 123456L; + long testVal2 = 9090909090L; + long testVal3 = 76543210234567L; + long testVal4 = 1L; + ArrayLongFPCache cache = new ArrayLongFPCache(); + cache.add(testVal1); + cache.add(testVal2); + cache.add(testVal3); + cache.add(testVal4); + assertTrue("should contain after add",cache.contains(testVal1)); + assertTrue("should contain after add",cache.contains(testVal2)); + assertTrue("should contain after add",cache.contains(testVal3)); + assertTrue("should contain after add",cache.contains(testVal4)); + } + + public void testReplacement() { + ArrayLongFPCache cache = new ArrayLongFPCache(); + for(long i=0; i<=ArrayLongFPCache.DEFAULT_SMEAR; i++) { + cache.add(i*cache.cacheLength()+1); + } + assertFalse("contains value after overwrite",cache.contains(1L)); + assertTrue("value not retained",cache.contains(cache.cacheLength()+1)); + + } + + public void testRemove() { + long testVal = 4516500024601L; + ArrayLongFPCache cache = new ArrayLongFPCache(); + cache.add(testVal); + cache.add(-testVal); + assertTrue("should contain after add",cache.contains(testVal)); + assertTrue("should contain after add",cache.contains(-testVal)); + cache.remove(testVal); + cache.remove(-testVal); + assertFalse("contains test value after remove",cache.contains(testVal)); + assertFalse("contains test value after remove",cache.contains(-testVal)); + } + +} diff --git a/commons/src/test/java/org/archive/util/fingerprint/LongFPSetCacheTest.java b/commons/src/test/java/org/archive/util/fingerprint/LongFPSetCacheTest.java new file mode 100644 index 00000000..5a9531cc --- /dev/null +++ b/commons/src/test/java/org/archive/util/fingerprint/LongFPSetCacheTest.java @@ -0,0 +1,96 @@ +/* LongFPSetCacheTest + * + * $Id$ + * + * Created Wed Jan 21 09:00:29 CET 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +package org.archive.util.fingerprint; + +import junit.framework.Test; +import junit.framework.TestSuite; + +/** + * JUnit test suite for LongFPSetCache + * + * @author James Casey + * @version $ Id:$ + */ +public class LongFPSetCacheTest extends LongFPSetTestCase { + /** + * Create a new LongFPSetCacheTest object + * + * @param testName the name of the test + */ + public LongFPSetCacheTest(final String testName) { + super(testName); + } + + /** + * run all the tests for LongFPSetCacheTest + * + * @param argv the command line arguments + */ + public static void main(String argv[]) { + junit.textui.TestRunner.run(suite()); + } + + /** + * return the suite of tests for LongFPSetCacheTest + * + * @return the suite of test + */ + public static Test suite() { + return new TestSuite(LongFPSetCacheTest.class); + } + + LongFPSet makeLongFPSet() { + return new LongFPSetCache(); + } + + /** + * This is a cache buffer, which does not grow, + * but chucks out old values. Therefore it has a different behaviour + * from all the other LongFPSets. We do a different test here. + */ + + public void testCount() { + LongFPSet fpSet = new LongFPSetCache(); + // TODO: for some reason, when run in a debugger, + // the cache-item-discard is glacially slow. It's + // reasonable when executing.) So, reducing the + // number of past-saturation tests in order to let + // full-unit-tests complete more quickly. + final int NUM = 800; // was 1000 + final int MAX_ENTRIES = 768; + + assertEquals("empty set to start", 0, fpSet.count()); + + for (int i = 1; i < NUM; ++i) { + fpSet.add((long) i); + assertEquals("correct num on add", + iJames Casey + * @version $ Id:$ + */ +public class MemLongFPSetTest extends LongFPSetTestCase { + /** + * Create a new MemLongFPSetTest object + * + * @param testName the name of the test + */ + public MemLongFPSetTest(final String testName) { + super(testName); + } + + /** + * run all the tests for MemLongFPSetTest + * + * @param argv the command line arguments + */ + public static void main(String argv[]) { + junit.textui.TestRunner.run(suite()); + } + + /** + * return the suite of tests for MemLongFPSetTest + * + * @return the suite of test + */ + public static Test suite() { + return new TestSuite(MemLongFPSetTest.class); + } + + LongFPSet makeLongFPSet() { + return new MemLongFPSet(); + } + + public void testFoo() { + + } +} + diff --git a/commons/src/test/java/org/archive/util/ms/15336-doc-preface.doc b/commons/src/test/java/org/archive/util/ms/15336-doc-preface.doc new file mode 100644 index 0000000000000000000000000000000000000000..759e49aba71aa2a74a84b66a82fbf2d02d1c258f GIT binary patch literal 28160 zcmeI5eT*DueaD}@^POKWeu?dvm*$=r#|g>Zk&qV!q9EJnbL_;i58Gfsh-P+oc6V~K zJIl`QoiDB0R5DWjXeCN$iz=joYLF@-ngZ<~Eh-=-QGuZ59|EKZ2&xdGl$sD6Xi2W0 z@AJ&;?VWSU1!z>n4*YI*=6V0Ue1E^+Gc(TP|GM=%AN{*mKV_C3Z!v@B`GxgnxaWR> z-*3#Xw-|FHzjg2Vg@py(z8>Od|M(5!zz;w61+z9-zs8vJ+dd~Dn!;9uT9DwszSo#b z%ss_>if6~pj=91mf7Y+rXFah$@kJL43!A(5R{EV+xZnL~4)|-Gdw=Wh z3teyb)!pBB5BiJmtX=uMiJ~7;u6B9-n>HBp4_t3r=iB|V=Lgptqv?N=vyRoA#pM5B z{1Y_1ZN!)}Tt7+s9qM4&k*`0W-?HMDoWJ_j@saGD_m`iS_nTk;7xj1cw9DJ)XVv>> z%9r=E{963!J@WQvS^HTE)*-)Bobz@oo%4CE)L%YE-v9D`^75B^&im`OW8KoHYya%V zFLdwmBiFw#QT_zi%iHN%?S_y$#$=>_I`Vpdj~=|(bzaZc`t|+I`Sk1S@#XcD+oR5B z!Ka*$%X{gqr?R}>8dE52Hf!Xd`p1eK7@4*w?36vU-%i@8iIG#aD78nUN@$acolN3R zz1T@D_mYOY71%UvMIqM}Ta1!wD`?hq*=&W)pcR&Fr%?`DwmxsuPLrCAcF2uJDIBvw zqiicl%eHHwttV;Q(x7eI!A#h&vtg?kw4=JMw3517?>-!*?W8rYan#^sGiXF6m8S3SlKoErzVK5tisHfLqMB6$Y(V(5QxP8bP_Np3?}a1Vwm34VtHjfc3YcAf~@X z$sp(;O<_oY$m|gt)M0cvEVh}JZ)wvR_~)9m=q70S8Q#^Ju!j+1h)Aa?XeSMOs+QD) zlzC5wty<7zv6{C6~TWNfK2VRQk$C5@npDvfNKjiSh|pFS~lX#eCC?U8jZq`6uG z50HpkcpV;~LGcXJ3yMjn?dFw>-`{E1f`*-}wIt;x6XAaQ3<}@2(!f$uhdoGI zO%n{T-i)y=1d?&QiCLDSW)NG(ZnVejTQRDw$;(R+mc1P_eE|4@yBN zMOt&U$mz1vkElIiPkRXmO=mrElFZmnlfk=SLKi_MZ(1jI#<6RWso@z0lR&Lk4G<-l zk<;aN)Nr$w)~bCyC^LW9HW$nz&Q@4zclEq0-V8`!vRLAJ7#~l?i7NlM+!w&1LINpa zkhnL~Nah;(dOHnw$pR(%%nWUrpLH=dU*1(I+5mhChu~Blu5~AeP|(G!Bilk>(V2ur zFudx!0miH_GP~g4M7Avczy4z;7nt z*93+i0=l#vyBP3MtxhYXI)9>Slym@y)52#lyqd&iTScE)m6JjYMJ5!!s1o z=s?n{MuC&xbkYp7@}o(ILACIy*a2M)jfkq!EtY-=AE~J1aM+OFq}It0?lo8$rBjV+ z>}AE+hr(7lYG@X!)>R^^?Z?wr;i}3e1Ed$n;rMv&cf|Ep2y-dHsW;TIdT?u#Wfnt+ z)wz14p=U@vnE-~*;9=FUT9js{4LSamTqRxYS z^-+aUUJXuSW^L=42H#>A2zDAs5-+8y^T?$!|7I`ZOxGPVoMx1hDsc+~ zI8~+JBu=XHnZPkOXTr6lBhO8!+No3`nTwALQd*{!Kr@@iVd53BGI0Y@q{2#>O))7e zo~@v_vcn8Nov9P>rEt2;G?AR#pJhmNY}1);v|>k?6v10&zckMpWwMoMTVWM*p%+&P zH%wU4!1XqeR4(R4_<(PnlF*lGQJka%G1*kTpfgirqyV&yVL3>I+`(|A5bEztZf zg4T!-4F+B#zRCt+tdt=0NG+B-tMHqkLeMU3h$(Z>QN=ET(u{hn&>u?a!xh5iF&n1M zutd-b*7y-Gs^fzV4{V@4GRg*VzRWE;b5S0E<6v&gMiq>7PSPv_33YS#3@(UX+#QmoE2ue zi$PUFObHcqI*~&%7|L|#ObY*I+z}nLUIG= z5pq=bGIJtBKERg}wsNN(kqM14;S_exGCNkyR6C9Fh`bDs$9#L-%w(-Ye&E;c++EIq z9*pvFC9@zRuDW=Eq>`xB@x$?!L9zHe+wdd$4&;v<(A8iEKBg`=kwtM?1RQkG0z7dy zlXO9)e0c-^%i7TPT!7vMjf`B=tHTacvN$?vDampI$r&|akD>i@M0ywk`QZ}i^_Ufr zQPrb3Xx-|V6X`~w5~Ww>?2plV)G0n1%{%qCAiCcon*1dCpJD5NF^Ar%Q|`r#h^K2!(`5) zwbB=9XcjnhjUZxcz&T3+4zZWJ$}}z})LUL3wMZ4H`QRyNyW~?Sga=3Pct;15@PW;o zfOK%VYG|DzI7&IWPE!CIa^%CBSYS;wvU341c3_ugPun;6}(eE#6Wm!(ODi+S@Y#Yz&CKn#nkpBrdoK-OCa$tGw%0H+9xtbZDS0E*6F4vLJ<$?iP}R1o7#3Oum)D zc~nP#(@03}8Zs>{PHCa4vR%~_?u~3ZvwVAn^iU;^@qUxbt6?uVhC&E8w z8&_xm0Ys18urs09I7<=<^hhO%_)6q2BsNU76tzm7y5GifLf6tjXO!2gG{Y?0yYQP0 zE>U20F6B`m04m78ma`}~u!MbX=dNdgUg_|XFvvOMa5ic?)dNd#5Hgqy>6PvHBpSCY zr{#AcW7zThNfslp!J z&|MfA5V~mg+Pn&cH~d=ECK@QkAkXJ!)}50{U7jS-Ug2WCq3OQOM>Lc9UW>**rYyvb zc@mYZE$+2xt{4$Ly|4f?e0C!^%NCF8C<(&MJqR<%IE5y6Vt_}zR++4e;nNu9*CaGK zf`vMOD;?vCSf&;=o2ro8)tY7V;NZEN(Ds@`<2-=TR-Q5~uI%?dDgV_Cg*O^yD z*;2M+=Gp{KtC@9D9A*2Q&{sZX`Let`NaBlbqgKOe_h_tZ22P_g7G;Zd>N;T$Vsl!k zgsa_SZT055;WbC|&E1AyHybykZp^d@?%wP0m+he}(?plW8*&$RBeL!OO2=(Uu$@Tw zfr%UjU#QdFe)FoBgRMghr*h1Ru*Rv4+xJc-{W&rq&u>|_+e6fNtU2#s40P^MlU{B) zuvr%F0BIv7B9w;r|IT6hrv{9KsSD*j$Y4xS6)5oR>J6e;GO3$iM_9o!+3RO|S+`DV zE`-yjsQB$R?epzOA4JGU1r_jA3+&w|!IxHg24TW?i5{JQI*`a}~gsVvKH%*!?25x?P+_ zQ;x$5p;`9KMtdTJFzBt@b=O{z_MhF(L(iHWa_7mV@-N0jD)`<^%-X$2%V(Jm7!ST` z#%%5Q?EzhMqZ2=WFDMoGofB{>j%F0mxb{lbybmOcL=YMlnF z8Etpy7%s7}{J9e8E{aaI7VF-Dp*7%_*#>u!eJvO* z;=J^j3s$%XP?4KDj#}&EfgjFL+qZQm!o^S(Rd7n@6JDR|EM-tkUhkrC&(&hcpbdnR zaa!}kHY6^|21xfB)8qJ^<$LAI5zsv?kOzVxfwpfP@B(-!1Ji679DYyhthU)4GAQ;@me12j}nR$GcQ4>~v z0N+(+e@S+DEGqG!RuLjII(|=B=I5d`7BNPTr;>TUu*?$1x+`+dkEO76o}sYzSLv|T zLH=G^f^7D^=Eujio}AGNUZhKekj@8FBtw+r65m0R83`|E9Srb{b9@uPS37Qi95MCK z7Gq`z9!7Q_Kq@HWwx7K+8UYiCN45zIG@t3{Ua(ipd3(SUVtb6u&U?E$qWGU3UCjA* zN&{~+aw|Y4dfY;z6+37%F59}&qP6S9-02>8)u`V0NJf5cs52Y3%qo9|<~wx5=C`tb zrEhMcGCfU%U>`Kyf8x~Px1F$ur;bja*oPWyEQ|G__6hHHe zd>;MtsyMK9XYH-quQfNX`=iM_wwM!#&zRmjJGs*^8+@I~J1*t@&HT47G9&r*E>lCy zuhM&-lK*DtQRqS5hAdMcT{s8r$n;_P*jsOb*}B<`Ji65k?c4a+ zjt}p8$K~6vE({D9v-Rw@!lmavWA5f`Hmc5$DeT)=SaN?PYx}(okL|dy>-(3V^KI46 z3~n~=yg_v~n!$g6?&;_EZA75%?%K^uM;P902KTt4U{7I?*R}WP#aCo5`+R=Mg!>o2 z4ICIK6biD}&s_$7gXKSg2Ep@x1j`?T&W#%LMQhA3SpIv^W$4LX=r-sH=xdO4=h3R(a9e{uTU<6%Xzi=r#Ue&%jIKf-vR6 zuA$L;F2;rRf`xTv;NuH`!V6=&)Kif_W`8`q25y?L&88b;U!stl15{?){Gs z?18R)=oMr`f2dcI1KZ7|W*eV-e9UZiUn}^rZ!~7#j~52`!bjnsKZeaQlbDvNo4~}} z!hum*KfHcT3bmoEV$bQI*}w1Ug^!wpJ^k$DDl`>(>CjKxSToJ<7F2bAo2JISLyXm+ zRL8qawC^VipXR)&`!@F`z6}UWksd;7FP)h+Z+`nmbJ+!a+>K8Ro5F^{*#8*fPN3VE zm04qc_pn)WwVB!X%)%GU)p*T(;%i-h)*LmFN!=U zpS`5;9%v&y`(Um^eD-d0f*!XO-rdvVwwLVDy6?EQ8vaL*iWZpj-TZ_vy}eJHlk~oO z$$VGS`wg&i1A3Lud9CAy=Bq>B2@Uy=9o+X~y1nKKeLGe#_7j1s7~`%7y4SvZuzT&x z*L1Ia`B3-Tmk)QZefiq%wJ%?{c+HR>M5Uh{CWa-1dW=t%o*};dr(NVOuk~{_NI`0H z34gst47%TIkhxYwNK81T((IEXeia~QUjQu_6 z!U#G-2}?zKoj2eEwlmP}$}eE_3brWf$?R&zHWwEVrXqS|?uAZu9&Os1P{0{9G|R zsk*r=yQgc?0Cx;gXA|@=psX*}ZsY5;gU}J^tqoM`PICc)%>cYRjD`mk%fJO_v!2p`>qYU7O(QV`eOGI^YFm81_xM=JLGPs z6|3#@E>?_w8-c|Pl$bgG%JF8t*|pHM4t>@=;QlWCO$23bAAY|o4Gatq4y_p)&@EB- z$ftISAgWFvlNyC-A>&zdf^ii_HmH3Q{UE6UMFdNZyaR7u1X=U6DbiBG=MCBD8;92n z=-}21j=rCeGGZjP*-gk-F!%2==O3qj&Gr9{5B$wu(cj+iz_6df=TVm(ui^7k?RwH; z(7cIreynqX$ftJ2pgEs&Ij_+_zkCjCgXde>i!9TWIK~{f;^p_TyoFTa0k0Dh7jpXc zx0ly>f$Q@<->9Sc3xBv0X%}{30i1BhrRH`sWsalSI$HwpDbk0d{Mrkhq~sWX$FkZR zNr9B!SaT<5r5GJkBJEl4QX!>9*4%;Cmax(O(dWSbAqSoZ;s-YrE*e31zVp7ndiv+b zYg<0?r|ZnMSAF^$*KtpvFQr%chaj2RM<7|+XCW-v{0k&B*R~#^g!w*%^EJ;vax?lS zn^3~YeW|ZKoJoDmKF&C2bCmNY&hOwX5H538R@LN;vo>cq3wH10jB_^kbJjfA%rb-) z?q74bed_p$+o$Y-siQ~jUVHNRv6Ex2O!L#BdK%*(PZaOuwO?L1zNoynuI9}fY`mwt z*mgST#P$weYRLLi{WoU62^P#K}a$^f}PyK%WDB4)i(D z=RltWeGc?F(C0v(1APws=5j!pt#V%F(8{xw$1C^M^LjmZ)^mD2n^*p>++2CLp2h1q zymEQv`Fe)0d|r9A-YZm2ue_fZt?fpf^C*L2CaTbVK%A*P{NhIu7Vnyu^KRFvSZ=ZC;m~ zhm)&6cAB9=K9bvqSubtkT?O}-&p3LH``+-x8+P!1!%f_~FS~zz&vzU>RagD<60|oT zU+2{{)GKCAc{!bC^(|gJ(+gQ8Uc=I>iurdZtLf|2e9a-7$>*?|HmallZi)_lJ)%+d zI;427?>;JSPP)_Pqxxo<7F=gf4hDckp4iX!^6w4TmF05J8b;J&K>>JLQZFn9k}_l X%vf)?P2?pLx4f9T^~?UBao~ReHBmfO literal 0 HcmV?d00001 diff --git a/commons/src/test/java/org/archive/util/ms/15336-doc-preface.txt b/commons/src/test/java/org/archive/util/ms/15336-doc-preface.txt new file mode 100644 index 0000000000000000000000000000000000000000..e1f7f8f6504c9e81886b239404c1003d75024763 GIT binary patch literal 16816 zcma*uTXR)Mk_F)N>Im~2Q1CLBu!)$Lj*00fns$$v#y}W){478Q#zJBc7vo<)`PEWV zwT}*XfFo_Q2+n7yHh{!j&HYjH+D~UFL$qYm-YRA z{r+WlTAwcJw=oyHv)%jM>)q4(X+*|d?9P`_Cw2X@zQ3)zr(;xRKdtW-q(o8Jm0;mPZ@E(dsd&ns}U#T%2DZgQF4xI z%*(oC{zd8Vu0CZ2pWfBzUx$pN-G}=9w(L2ntFLM_32$qTPg(oc?s(YwZb+t0?CBLA z56S0a%*(pVOXs`)Ei2e(y;t@9sJ#4oto5pNKH1&g-KuNxwzZFTABX&3$ETCww~wXC ztGX9^k7}OCoR)@X!z&-hU2F3BhjH&?eWIyV&qlQ9z~`h)za+-{HeJ??lPjNm+W7Un zWX5-7zbNTE$6N2mI&^qe|1L)K-Ys!&KZ)pR#T9P09(-QXF2~$NLqv?~n3CEa6`S1M z%6L$++5Ypme^%>2#p}|B*N#d@(I7S6nHKO}R-oZ|jiIHvr}h7$bc3g?b~>yiZ(aR$ zMN=&A4G+DnF_&ZYSPtDmWsv%`zFX(0{-4&$(F^)vE83akinyJZw|Tm~aZ=jRFFIeA ze#w=SLUC{r8Cjh*Y)Sm#>7+(Qzm~Y>Ofs4UB*TtTv^*;NXdyEo+Nw|Me!R7=DgS@1 zG4MC_W)p4Lm6*hbKh}5tJFa^&(N9QwRKJr!qJK1aNnX+|{(dyZ^7os%*4Q@7s`-t- z;9`vz{JNXG3$I+(i1WI_*JRz;eLFJhRoQX0`=+${sn`-%-mTwRKY6{MNgoSua)e&V zkr{D#ieCJBHdsh3#ZKn1yQ8w7{p%R9H4fXnEQyKKMfm~i!Ry_Uw{o>?n3;Gz-`Yd_ zEG6o+dR=S0DCs_nqTHA=*>rP*@@QyuR*ls`SF#-1&q^wOlbsovm}H#%n0&^I);_5@ z`#uwcWFd?`AO4?=v=wcS{DZ{PfrUF2KL`qb zX=g>!JDSOla7_@-4iP-v{Z`U30&iVhX(qcd&{0VaFQlF&>y3Y1(Z<&5Rx)IZadp)S z&+4bzcQ)2yx0UF|4=YYtV9iuQd0yUsN>p32*9CvS5pLey0V0uldd9WaqCnuhGS_eWZ#? zEDOJpw6aEe6eC0=Xv)Zr5Z_NpFbD`@VE-OsIb93J(Q!bQRasZxNY&P)Cu^WDk#T_h4_+hR0V@9N#3r}#*&*>n3zOrrd z)Qh^Sca-bl=frB}0u7t3#pSToU-oS z=9HFsw{-TI4fr)(%=O;oo4Whkxcj$Zr8#`OW?5HTi~);5Xn1b=%e^3gA23PpvS4Ad zk~oo~vwL}kt^KS|C9+OXF;~G`I1v(CYAWg55z8>?+oj$@Kytf%qi;w(?doPs-^GSO zu)AXKcNkJv_^#GTWMY%m*`3b&e0&S<+-0s z>RGn)@7GniAI~IqKMa}l*3rpfR>3bVWfD!C@MIs^XC=pDM(;Nx3kwB%Eh}?>7`CV& zc6`YzpGakeZbgRVTBGhd6Ga;I(*@7ctmhezKeA^$Enj|LSLU9m<@(J=$e~j>M_!Yb zHG1rf+lnQfQF!24Nm2=#Jub(pla+1`w&m}RjHvMJJWt_=iSk}KJMysUmIL3_=%B?| zeQ{!kUAE^fv{w_XaHxg`F*txEbtIPE91QFIJ{F@>JPet{k2Bxgjss4B#W6$ z;+LLdZwh~66@ND(W^AmK$|Apm8$DujGJC0(`l(hi{=EG9A2n-c@s`&oGA3u(d%h@y z@n>oJpwy?m@COFKdLexcCe**qFUy zYFT3=3~QcI(UU}ZC$H1nkfuxHV>^b-iY;qZ*V&s9O#3WG`UpGa?Ed7a6~bfRjxJ=s zKY^=A&&k`i{$};Uvwg&Kbo#cf4O$*r8^Kt(nQdMSe8by8jmOhT$ z>{!=0TQOCSrt9t&e&tJQ8%#U(^E;S0q<`1p?6+a8>qYW1{$x{UCwW^uiFkVRABUY} zX8#Zb{_D^QUb_y4w>nm`CtfER0z~&iUDc_x$4NzZ?Li?dWL{*NsXoCjUpr|C>SP}M zAh>tZIbF%=G7jV)9&kN*GxcdL|47byvaxn-=4XGjZ%OZuf}lhi=3(T?K)D&+0jr!J zM=iI*P9=5D7VL;EOJIQR*-yY8uZSuh<*PNrIv%{d^?S2Tm>I4}-7;yahg#+ntk3?s zIsIiJxN#YNi{#bDzRznSmi`pXicq3FTbNluyz=MnN6Zde;tkChkFC9(PZX&`gC=~uFMGmiSfX7Cbeqv zEH%1oRPZwE>GSd^KFl*A9^m22(eu2mEF}NMriXO@`*Gd#GWKK)e60OTd(COLUdh?v z8uy9q_G=?xW!>3Ft>=~Tg(xPneZG(r2Q9L4J&W6(0js7wB2UBsmbZQ8x z(Uym2=aX8*M|vn4@d1ryhU_CsXe)oPyLK1DSVyx8^k2f^rBPFU*C;P z-qs_l!V*lS>lMZC#-3Rpx9^#`{>p!of!B@{l0@E4T&7?goJ7}~^{t;j&q$HZAG0Iz zc8RlfowxnK;Z^w?t|m9}MEbPopnGFk*H}!zSG}UB;cL62w^x~+dyW6e#)t5fH$Lc| z3YE$$;(68)9^aoSmP_g2PkJ!G=`E}RO zl_qS#f5B$QFVZ~i?I)k@)9k?^NdGKzSed)Q1NMT8JROu>xqp~7I-c`cMEj;sr~3&~ zYk;vJP)zqNBn<YkN+IAjsP%GLyut)Vm)(;Lp_J+3`UZ+4iu! z0h@NO_GxCkuba8I#_KVX+|B770F-i;-7y0;ClP+ zxMZfnJ+2YkCmHjccV8OZvn~9MYan`iCimspdf(LfuF7A2;*#A&kJ~=+I{N&)%UbRA zaLd!eLiYw@VZ}s>Cv+T`bXzGOT-ot+UB?cwFPgT$9**n$+;0p8#V{WJm4O}QxnjK4 z%tbR=wNxnK{mw%DgwEmXZ?8ZnYtkjOtnjZD_ABJ`vRXz)tNs?oX%BxW+dmE7&bu@$ zy*G5l{xA>8bH}an$oe3`X%h*mIgG-7@2J(!)$8qx^fzosN1Un~zTtbHf)^|0`KhJE?LJ&@SnR1L`%=$m4O8{JZ`W*_oTWRzH4Etp>?!Ph zS=ema+%XYAJ6=oWnpJ>JFY0Qv)t$+E`IjulVROHLf$_G;=OpII@QeA_Nc7Vy(1c$- z7oo53ozFS#b3M;W4o8q@JLkljwBRF>`Ak2=>dpuGmUnLCS}F|7W+xUz^KEx-JdH7W zr`4TvB5uXRoc8hCVSU+XG>=Zd2k+Sxq%yENJ?!=jD7>=n+hYwsWZgq-%>Vqm&chS5 z;Iqt>8Ixy%$;`7i&S_7N=&?b7u{3DkvyhKf#NHFRc8Hs@LQZ^B_s*_HZnc{|u9_IO zVY`{@Zg_Hm*tqwP8WkQ89sXf8>A`5*Z$Fu5ArpzJU3SXX-@cvaVyQ&YNpA4c|0e_G z!_3{;+0L3x62HE#f3!_zs)XwvBtF<;Dx8e(((yC~^t(pRb9f9A=5J55=lJieR!)bT zw?6t_&Smf&c4~=}Gk9L7?@d$r1RwugC^94b(e+&CkDYWjTO0ILD;M4Rd=mL+jQoAX z^1FJf^Di~GHCq|6>r(e!-L>^PzPIl2^+oZ7XH;~~sc$MY{l6Y9BRW5_|F>JhIvB65 z6A+^ zv4-vjZtOGJBHLBmaKfyP;pLVC4Kjts-ERgp&iwjxLJiT&v-GgRKHgz0{bi%KVkNj3+MaZ_A1GJwIR<=CcZO!#dGxPIPYt~mtSU+FJWu4+1F2Z z(xu%P7U!wZy8lnV75yHUwNR?Aun8~ESgoCmsH&Y9BiT98MX#&_7ow8>CQ%l(ZMAU% zk(!zJ`r|$EncBLRT+>M>lX%Tqjc?IP)~pO})S8>Uk2TbH8;MF4recoI}oxn`+@M zbWMgZVs^z{iB|tdLUN(Kz?;4AOFzXUDn@)7o2<_g`$JWr_krS*4st&0%DnWGv)=I! ztn+zxMcu2Z?8#MUuD;QbHQn`24iTxGYsx>K>$4Kf*l*k6 zocvW0Kj>up{6Dw}YpHR(&P)7}XM)+8=tu2!Tu<(2SF-l#-sh#g>S$cn!d>Jymtygp zY-V?JKH@+QtURfhiRO9f;_vYIEUS{C>hfsV*!R^BJ`DRsUuZOpFaW3-`Ggmct4xi-zkH-dRy5@(v-0HlEjuS>mu_ zm9Jtcq{|#*U@e$#|D|qpc6HvQOJ6x=pE+5VdWYXKPyXSZ_-&%z6eShe?d-oE)`gwq56SBeB9>eGUd4Fpv9BTX;n<_e5LMmEf2mLQ zT|C2^T@N4Bce_mSbZv6%!`=T3pXaLlY(DMWn3`jN`L>u9aY|6P3}W&@_)^(?y}7jIWsyw`-tw77uI+E;e#!!+$*fmHkhB4B0Y(> zn8GLf76k#(b+t_9vNpVl$$U2Y?yz?L9k@{wDY3%n!=4ZBatgF&OU_7~Or~SXIyzSt zS)={q{NVS(fgNij=f4Lo=dcz5c!j@s%Q zU-DFYRF}-l>CBh|I*#ik?x*^GTz~y~zy2Eb@zve?<$+{yc=KUphg?c$ zVx@oE{rm1u^ 0) { + throw new IOException(errors + " errors, see stdout."); + } + } + + + private int runDoc(File doc) throws IOException { + System.out.println("===== Now processing " + doc.getName()); + String name = doc.getName(); + int p = name.lastIndexOf('.'); + String expectedName = name.substring(0, p) + ".txt"; + File expectedFile = new File(TEST_DIR, expectedName); + if (!expectedFile.exists()) { + createExpectedOutput(doc, expectedFile); + } + return runFiles(doc, expectedFile); + } + + + private void createExpectedOutput(File doc, File output) + throws IOException { + FileInputStream finp = new FileInputStream(doc); + FileOutputStream fout = new FileOutputStream(output); + + try { + WordDocument wd = new WordDocument(finp); + Writer writer = new OutputStreamWriter(fout, "UTF-16BE"); + wd.writeAllText(writer); + } finally { + close(finp); + close(fout); + } + } + + + private static void close(Closeable c) { + try { + c.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + + private int runFiles(File doc, File expected) + throws IOException { + FileInputStream expectedIn = new FileInputStream(expected); + Reader expectedReader = new InputStreamReader(expectedIn, "UTF-16BE"); + Reader docReader = Doc.getText(doc); + try { + return runReaders(docReader, expectedReader); + } finally { + close(docReader); + close(expectedReader); + } + } + + + private int runReaders(Reader doc, Reader expected) + throws IOException { + int count = 0; + int errors = 0; + boolean go = true; + while (go) { + int ch = doc.read(); + int expectedCh = correctPOI(expected.read()); + if ((ch < 0) || (expectedCh < 0)) { + go = false; + if ((ch >= 0) || (expectedCh >= 0)) { + errors++; + System.out.println("File lengths differ."); + } + } + if (ch != expectedCh) { + errors += 1; + report(count, expectedCh, ch); + } + count++; + } + return errors; + } + + + private void report(int count, int expected, int actual) { + StringBuilder msg = new StringBuilder("#").append(count); + msg.append(": Expected "); + msg.append(expected).append(" (").append(toChar(expected)); + msg.append(") but got ").append(actual).append(" ("); + msg.append(toChar(actual)).append(")."); + System.out.println(msg); + } + + + private static String toChar(int ch) { + if (ch < 0) { + return "EOF"; + } else { + return Character.toString((char)ch); + } + } + + /** + * Corrects POI's Cp1252 output. There's a bug somewhere in POI that + * makes it produce incorrect characters. Not sure where and don't have + * time to track it down. But I have visually checked the input + * documents to verify that Doc is producing the right character, and + * that POI is not. + * + * @param ch the POI-produced character to check + * @return the corrected character + */ + private static int correctPOI(int ch) { + switch (ch) { + case 8734: + // POI produced the infinity sign when it should have + // produced the degrees sign. + return 176; + case 214: + // POI produced an umat O instead of an ellipses mark. + return 8230; + case 237: + // POI produced an acute i instead of a fancy single quote + return 8217; + case 236: + // POI produced a reverse acute i instead of fancy double quote + return 8220; + case 238: + // POI produced a caret i instead of fancy double quote + return 8221; + default: + return ch; + } + } + + +} diff --git a/commons/src/test/java/org/archive/util/ms/PieceReaderTest.java b/commons/src/test/java/org/archive/util/ms/PieceReaderTest.java new file mode 100644 index 00000000..6b160749 --- /dev/null +++ b/commons/src/test/java/org/archive/util/ms/PieceReaderTest.java @@ -0,0 +1,156 @@ +/* PieceReaderTest +* +* Created on September 18, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.util.ms; + + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Random; + +import org.archive.io.ArraySeekInputStream; +import org.archive.io.SafeSeekInputStream; +import org.archive.io.SeekInputStream; + +import junit.framework.TestCase; + + +/** + * Unit test for PieceReader. Takes a quatrain of a sonnet and stores the + * lines out-of-order, then constructs a PieceTable that will re-order the + * lines correctly. Finally constructs a PieceReader with that raw data + * and piece table and sees if the correct quatrain is produced by the + * stream. Also performs some tests of random seeking within the stream. + * + * @author pjack + */ +public class PieceReaderTest extends TestCase { + + + final private static String[] QUATRAIN = new String[] { + "If the dull substance of my flesh were thought\n", + "Injurious distance could not stop my way\n", + "For then, despite of space, I would be brought\n", + "From limits far remote where thou dost stay.\n" + }; + + + final private static String QUATRAIN_STRING = + QUATRAIN[0] + QUATRAIN[1] + QUATRAIN[2] + QUATRAIN[3]; + + final private static byte[] QUATRAIN_BYTES; + final private static byte[] PIECE_TABLE; + + + + public void testPosition() throws Exception { + PieceTable table = makePieceTable(); + SeekInputStream asis = new ArraySeekInputStream(QUATRAIN_BYTES); + asis = new SafeSeekInputStream(asis); + PieceReader reader = new PieceReader(table, asis); + StringBuilder sb = new StringBuilder(); + for (int ch = reader.read(); ch > 0; ch = reader.read()) { + sb.append((char)ch); + } + assertEquals(QUATRAIN_STRING, sb.toString()); + + reader.position(0); + sb = new StringBuilder(); + for (int ch = reader.read(); ch > 0; ch = reader.read()) { + sb.append((char)ch); + } + assertEquals(QUATRAIN_STRING, sb.toString()); + + Random random = new Random(); + for (int i = 0; i < 1000; i++) { + int index = random.nextInt(QUATRAIN_BYTES.length); + reader.position(index); + char ch = (char)reader.read(); + assertEquals(QUATRAIN_STRING.charAt(index), ch); + } + } + + + private static PieceTable makePieceTable() throws IOException { + ArraySeekInputStream stream = new ArraySeekInputStream(PIECE_TABLE); + int maxSize = QUATRAIN_BYTES.length; + return new PieceTable(stream, 0, maxSize, 4); + } + + + static { + QUATRAIN_BYTES = new byte[QUATRAIN_STRING.length()]; + PIECE_TABLE = new byte[4 * 12 + 5 + 4]; + int ofs = 0; + int line3 = 0; + ofs += addLine(ofs, QUATRAIN[2]); + int line1 = ofs; + ofs += addLine(ofs, QUATRAIN[0]); + int line4 = ofs; + ofs += addLine(ofs, QUATRAIN[3]); + int line2 = ofs; + ofs += addLine(ofs, QUATRAIN[1]); + + int start = 0; + int end = QUATRAIN[0].length(); + addPiece(0, start, end, line1); + + start += QUATRAIN[0].length(); + end += QUATRAIN[1].length(); + addPiece(1, start, end, line2); + + start += QUATRAIN[1].length(); + end += QUATRAIN[2].length(); + addPiece(2, start, end, line3); + + start += QUATRAIN[2].length(); + end += QUATRAIN[3].length(); + addPiece(3, start, end, line4); + + ByteBuffer buf = ByteBuffer.wrap(PIECE_TABLE); + buf.order(ByteOrder.LITTLE_ENDIAN); + buf.put(0, (byte)2); + buf.putInt(1, 52); + } + + + private static int addLine(int ofs, String line) { + for (int i = 0; i < line.length(); i++) { + QUATRAIN_BYTES[ofs + i] = (byte)line.charAt(i); + } + return line.length(); + } + + + private static void addPiece(int index, int start, int end, int fp) { + ByteBuffer buf = ByteBuffer.wrap(PIECE_TABLE); + buf.order(ByteOrder.LITTLE_ENDIAN); + int orig = fp; + fp = (fp * 2) | PieceTable.CP1252_INDICATOR; + if ((fp & PieceTable.CP1252_MASK) / 2 != orig) { + throw new RuntimeException("No."); + } + buf.putInt(index * 4 + 5, start); + buf.putInt(5 + 20 + index * 8 + 2, fp); + } +} diff --git a/commons/src/test/java/org/archive/util/ms/X.doc b/commons/src/test/java/org/archive/util/ms/X.doc new file mode 100644 index 0000000000000000000000000000000000000000..04e2a47b87d5061621cf107e1427ea821fa68ad4 GIT binary patch literal 19456 zcmeHPU2GIp6h1Ti(=HS!wH5?~h1Nh~Q>`K)M$Pi0B9WG0wS6*fOKVG8wr;mUe6bIh z^sz=0(GV#QM2yDNKr|*K!VBTW$b%U5!Juy@XbKV|iFEzGyK{G^16y_#jnZ?Hv*+A% z?>YD0Z|+Rz%$@1EFBe}p^6|1CBtVZzNT#zfiTdmg^wqXpEm9Aiw&`p(OLH6yRq20( zz{R7dr6L&%i%ivg&Vwk)oOG}R38MW6J zve`v>V=>(}7&A%}mlufAuUlJC*3#h7ZqIjY7FmV1Z=#;kwr|BWK0*0I&+o6wGw>%O zXT3P3NP-**iR^-7dqiX>%2(ijCuEQ;9(H`edTTEx8JEOyl59LHtzWLTfJY2ttAo+{ z1ttA(amdHBae{HJpN((Jo8iyn_#7k_S3ZvQ4@w)~`Um3$%RzrzU#<-HH-YhT9lg_^ zIPH&R*WXFh&qGDf&z4sK*ZWb=^+&Sy_plCTE8BLtR=4ig;&kgGmKVvMkCY8?wKxW& z_}j70whN2nERrz1m5L+Kv7mUJ*31C}yub5*%z>%u6blGM>v|8b{JC(ScdbyR+T1{Z ze;M=l`O|Y_&N@pU0=YOdY_;lhJWDa@}Ys5{|euk@IAS(*}44aoC212d*3a z#c^m;W`XTR>xU2rBXq?}fUgSxdcDt&hFua&H|%{A7Fi#b4t?T) z@KWX2Qkgo3_Pu6+pMLj#4;j^Y0I9RY24M+Rn})__t=95WNTw{xUIP^+BXBf&s^Wy4 zz~N1=FWIT&hFlWw=GFuc7mkyEg%=Go<$HRGm7jgJ7}xJug8CXSkRu(Q7Bd$(UhLws z9V@R3=h=;HMMt0&ihv@Z2q*%IfFhs>C<2PW-$h^=iCk!&ORB*vvA+Swz!$;1J^mfc z9no3@tAdO}ayzOKlIitENWL+UfMlQBAn$|I>q@7FQ$3k@dn&y<-rU^ufFxkIl%R4sy5RXA z&p3Xa_+Z%iTr`(|6w0VlQUnwMML-cy1QY>9KoL*`6ahs*5l{sFF$8AI|K0OnoIljG zy!wr|kpDMc{fPPhS77d&Gnah<%sjsr%>C&h@M7@G;3eQuFx!rSnZJ*NnZF+d^9}L| zF!TLmVCLR@>)z(|{(L!@H%Hz1CN7#6l`GL9{TQ!9s!*cFET1E7>&pyun_Qf2loSC) zKoL*`6ahs*5l{pa0YyL&Py`f#+Z_R$BQxLTK05PxX2{&P=l^x!em(c-na49nXTHw8 zdG5=zp7}oa@|o9jf1F>cGq-2{&-%L{mw}gqSAg#ZGYek@X8kqr8c%1Ltn}|8z!&aQ z=Ie{c&36`Qyi8vX)k(y$qvaZk^4|l1PMKePzTu~rTK3iBy@&?bMm>A8@B0mZ(^BP zfxYsp8e;zxjzW(a6D=8hqQLK8iqksuPsDldmO6jab-w@L%*z|kM)YszUTXhKIUTQm zN48*wgLrn>icdK9u%;}kLI1pl@tRd?|LlP83z2fYcH`J*@OnPa1-uMY2G!vy<4x!9 zt$VSujQFKAy=nim1R-{M&GjQVw{!DuH1NLL)>PNXVE5jPSu-u0H?;FK U=9_R$wx((0EUQhc{%Z*Q35k{Tq5uE@ literal 0 HcmV?d00001 diff --git a/commons/src/test/java/org/archive/util/ms/X.txt b/commons/src/test/java/org/archive/util/ms/X.txt new file mode 100644 index 0000000000000000000000000000000000000000..6b827fb702df3069544b6b5faea2d47209a5f934 GIT binary patch literal 4 LcmZRGVBiG+0T}>h literal 0 HcmV?d00001 diff --git a/commons/src/test/resources/org/archive/settings/path/anonymous.resolved.txt b/commons/src/test/resources/org/archive/settings/path/anonymous.resolved.txt new file mode 100644 index 00000000..922fecc6 --- /dev/null +++ b/commons/src/test/resources/org/archive/settings/path/anonymous.resolved.txt @@ -0,0 +1,257 @@ +anonymous,o1|root=map, java.lang.Object +anonymous,global|root:first=object, org.archive.settings.path.Foo +default|root:first:eight=biginteger, 8 +default|root:first:eleven=pattern, ^11$ +default|root:first:five=int, 5 +default|root:first:four=float, 4.0 +default|root:first:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:first:nine=bigdecimal, 9.0 +default|root:first:one=byte, 1 +default|root:first:seven=short, 7 +default|root:first:six=long, 6 +o1|root:first:ten=string, three plus seven +default|root:first:three=double, 3.0 +default|root:first:twelve=file, /12 +default|root:first:two=char, 2 +default|root:first:zero=boolean, false +anonymous,global|root:primary=primary, org.archive.settings.path.Foo +default|root:primary:eight=biginteger, 8 +default|root:primary:eleven=pattern, ^11$ +default|root:primary:five=int, 5 +default|root:primary:four=float, 4.0 +default|root:primary:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:primary:nine=bigdecimal, 9.0 +default|root:primary:one=byte, 1 +default|root:primary:seven=short, 7 +default|root:primary:six=long, 6 +default|root:primary:ten=string, ten +default|root:primary:three=double, 3.0 +default|root:primary:twelve=file, /12 +default|root:primary:two=char, 2 +default|root:primary:zero=boolean, false +anonymous,o1|root:second=object, org.archive.settings.path.Foo +default|root:second:eight=biginteger, 8 +default|root:second:eleven=pattern, ^11$ +default|root:second:five=int, 5 +default|root:second:four=float, 4.0 +default|root:second:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:second:nine=bigdecimal, 9.0 +default|root:second:one=byte, 1 +default|root:second:seven=short, 7 +default|root:second:six=long, 6 +default|root:second:ten=string, ten +default|root:second:three=double, 3.0 +default|root:second:twelve=file, /12 +default|root:second:two=char, 2 +default|root:second:zero=boolean, false +anonymous,global|root:bar=object, org.archive.settings.path.Bar +global|root:bar:foo=object, org.archive.settings.path.Foo +default|root:bar:foo:eight=biginteger, 8 +default|root:bar:foo:eleven=pattern, ^11$ +default|root:bar:foo:five=int, 5 +default|root:bar:foo:four=float, 4.0 +default|root:bar:foo:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:foo:nine=bigdecimal, 9.0 +default|root:bar:foo:one=byte, 1 +default|root:bar:foo:seven=short, 7 +default|root:bar:foo:six=long, 6 +default|root:bar:foo:ten=string, ten +default|root:bar:foo:three=double, 3.0 +default|root:bar:foo:twelve=file, /12 +default|root:bar:foo:two=char, 2 +default|root:bar:foo:zero=boolean, false +default|root:bar:foo-auto=auto, root:primary +anonymous,global|root:bar:list=list, org.archive.settings.path.Foo +anonymous,global|root:bar:list:0=object, org.archive.settings.path.Foo +default|root:bar:list:0:eight=biginteger, 8 +default|root:bar:list:0:eleven=pattern, ^11$ +default|root:bar:list:0:five=int, 5 +default|root:bar:list:0:four=float, 4.0 +default|root:bar:list:0:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:list:0:nine=bigdecimal, 9.0 +default|root:bar:list:0:one=byte, 1 +default|root:bar:list:0:seven=short, 7 +default|root:bar:list:0:six=long, 6 +default|root:bar:list:0:ten=string, ten +default|root:bar:list:0:three=double, 3.0 +default|root:bar:list:0:twelve=file, /12 +default|root:bar:list:0:two=char, 2 +default|root:bar:list:0:zero=boolean, false +anonymous,global|root:bar:list:1=object, org.archive.settings.path.Foo +default|root:bar:list:1:eight=biginteger, 8 +default|root:bar:list:1:eleven=pattern, ^11$ +default|root:bar:list:1:five=int, 5 +default|root:bar:list:1:four=float, 4.0 +default|root:bar:list:1:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:list:1:nine=bigdecimal, 9.0 +default|root:bar:list:1:one=byte, 1 +default|root:bar:list:1:seven=short, 7 +default|root:bar:list:1:six=long, 6 +default|root:bar:list:1:ten=string, ten +default|root:bar:list:1:three=double, 3.0 +default|root:bar:list:1:twelve=file, /12 +default|root:bar:list:1:two=char, 2 +default|root:bar:list:1:zero=boolean, false +anonymous,global|root:bar:list:2=object, org.archive.settings.path.Foo +default|root:bar:list:2:eight=biginteger, 8 +default|root:bar:list:2:eleven=pattern, ^11$ +default|root:bar:list:2:five=int, 5 +default|root:bar:list:2:four=float, 4.0 +default|root:bar:list:2:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:list:2:nine=bigdecimal, 9.0 +default|root:bar:list:2:one=byte, 1 +default|root:bar:list:2:seven=short, 7 +default|root:bar:list:2:six=long, 6 +default|root:bar:list:2:ten=string, ten +default|root:bar:list:2:three=double, 3.0 +default|root:bar:list:2:twelve=file, /12 +default|root:bar:list:2:two=char, 2 +default|root:bar:list:2:zero=boolean, false +anonymous,o1|root:bar:list:3=object, org.archive.settings.path.Foo +default|root:bar:list:3:eight=biginteger, 8 +default|root:bar:list:3:eleven=pattern, ^11$ +default|root:bar:list:3:five=int, 5 +default|root:bar:list:3:four=float, 4.0 +default|root:bar:list:3:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:list:3:nine=bigdecimal, 9.0 +default|root:bar:list:3:one=byte, 1 +default|root:bar:list:3:seven=short, 7 +default|root:bar:list:3:six=long, 6 +default|root:bar:list:3:ten=string, ten +default|root:bar:list:3:three=double, 3.0 +default|root:bar:list:3:twelve=file, /12 +default|root:bar:list:3:two=char, 2 +default|root:bar:list:3:zero=boolean, false +anonymous,o1|root:bar:list:4=object, org.archive.settings.path.Foo +default|root:bar:list:4:eight=biginteger, 8 +default|root:bar:list:4:eleven=pattern, ^11$ +default|root:bar:list:4:five=int, 5 +default|root:bar:list:4:four=float, 4.0 +default|root:bar:list:4:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:list:4:nine=bigdecimal, 9.0 +default|root:bar:list:4:one=byte, 1 +default|root:bar:list:4:seven=short, 7 +default|root:bar:list:4:six=long, 6 +default|root:bar:list:4:ten=string, ten +default|root:bar:list:4:three=double, 3.0 +default|root:bar:list:4:twelve=file, /12 +default|root:bar:list:4:two=char, 2 +default|root:bar:list:4:zero=boolean, false +anonymous,o2|root:bar:list:5=object, org.archive.settings.path.Foo +default|root:bar:list:5:eight=biginteger, 8 +default|root:bar:list:5:eleven=pattern, ^11$ +default|root:bar:list:5:five=int, 5 +default|root:bar:list:5:four=float, 4.0 +default|root:bar:list:5:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:list:5:nine=bigdecimal, 9.0 +default|root:bar:list:5:one=byte, 1 +default|root:bar:list:5:seven=short, 7 +default|root:bar:list:5:six=long, 6 +default|root:bar:list:5:ten=string, ten +default|root:bar:list:5:three=double, 3.0 +default|root:bar:list:5:twelve=file, /12 +default|root:bar:list:5:two=char, 2 +default|root:bar:list:5:zero=boolean, false +anonymous,o2|root:bar:list:6=object, org.archive.settings.path.Foo +default|root:bar:list:6:eight=biginteger, 8 +default|root:bar:list:6:eleven=pattern, ^11$ +default|root:bar:list:6:five=int, 5 +default|root:bar:list:6:four=float, 4.0 +default|root:bar:list:6:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:list:6:nine=bigdecimal, 9.0 +default|root:bar:list:6:one=byte, 1 +default|root:bar:list:6:seven=short, 7 +default|root:bar:list:6:six=long, 6 +default|root:bar:list:6:ten=string, ten +default|root:bar:list:6:three=double, 3.0 +default|root:bar:list:6:twelve=file, /12 +default|root:bar:list:6:two=char, 2 +default|root:bar:list:6:zero=boolean, false +anonymous,o1|root:bar:map=map, org.archive.settings.path.Foo +anonymous,global|root:bar:map:a=object, org.archive.settings.path.Foo +default|root:bar:map:a:eight=biginteger, 8 +default|root:bar:map:a:eleven=pattern, ^11$ +default|root:bar:map:a:five=int, 5 +default|root:bar:map:a:four=float, 4.0 +default|root:bar:map:a:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:map:a:nine=bigdecimal, 9.0 +default|root:bar:map:a:one=byte, 1 +default|root:bar:map:a:seven=short, 7 +default|root:bar:map:a:six=long, 6 +default|root:bar:map:a:ten=string, ten +default|root:bar:map:a:three=double, 3.0 +default|root:bar:map:a:twelve=file, /12 +default|root:bar:map:a:two=char, 2 +default|root:bar:map:a:zero=boolean, false +anonymous,o1|root:bar:map:b=object, org.archive.settings.path.Baz +default|root:bar:map:b:eight=biginteger, 8 +default|root:bar:map:b:eleven=pattern, ^11$ +default|root:bar:map:b:five=int, 5 +default|root:bar:map:b:four=float, 4.0 +default|root:bar:map:b:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:map:b:nine=bigdecimal, 9.0 +default|root:bar:map:b:one=byte, 1 +default|root:bar:map:b:seven=short, 7 +default|root:bar:map:b:six=long, 6 +default|root:bar:map:b:ten=string, ten +default|root:bar:map:b:three=double, 3.0 +default|root:bar:map:b:twelve=file, /12 +default|root:bar:map:b:two=char, 2 +default|root:bar:map:b:zero=boolean, false +anonymous,global|root:bar:map:c=object, org.archive.settings.path.Foo +default|root:bar:map:c:eight=biginteger, 8 +default|root:bar:map:c:eleven=pattern, ^11$ +default|root:bar:map:c:five=int, 5 +default|root:bar:map:c:four=float, 4.0 +default|root:bar:map:c:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:map:c:nine=bigdecimal, 9.0 +default|root:bar:map:c:one=byte, 1 +default|root:bar:map:c:seven=short, 7 +default|root:bar:map:c:six=long, 6 +default|root:bar:map:c:ten=string, ten +default|root:bar:map:c:three=double, 3.0 +default|root:bar:map:c:twelve=file, /12 +default|root:bar:map:c:two=char, 2 +default|root:bar:map:c:zero=boolean, false +anonymous,o2|root:bar:map:e=object, org.archive.settings.path.Foo +default|root:bar:map:e:eight=biginteger, 8 +default|root:bar:map:e:eleven=pattern, ^11$ +default|root:bar:map:e:five=int, 5 +default|root:bar:map:e:four=float, 4.0 +default|root:bar:map:e:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:map:e:nine=bigdecimal, 9.0 +default|root:bar:map:e:one=byte, 1 +default|root:bar:map:e:seven=short, 7 +default|root:bar:map:e:six=long, 6 +default|root:bar:map:e:ten=string, ten +default|root:bar:map:e:three=double, 3.0 +default|root:bar:map:e:twelve=file, /12 +default|root:bar:map:e:two=char, 2 +default|root:bar:map:e:zero=boolean, false +anonymous,o1|root:bar:map:d=object, org.archive.settings.path.Foo +default|root:bar:map:d:eight=biginteger, 8 +default|root:bar:map:d:eleven=pattern, ^11$ +default|root:bar:map:d:five=int, 5 +default|root:bar:map:d:four=float, 4.0 +default|root:bar:map:d:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:map:d:nine=bigdecimal, 9.0 +default|root:bar:map:d:one=byte, 1 +default|root:bar:map:d:seven=short, 7 +default|root:bar:map:d:six=long, 6 +default|root:bar:map:d:ten=string, ten +default|root:bar:map:d:three=double, 3.0 +default|root:bar:map:d:twelve=file, /12 +default|root:bar:map:d:two=char, 2 +default|root:bar:map:d:zero=boolean, false +anonymous,global|root:bar:slist=list, java.lang.String +anonymous,global|root:bar:slist:0=string, zero +anonymous,global|root:bar:slist:1=string, one +anonymous,global|root:bar:slist:2=string, two +anonymous,o1|root:bar:slist:3=string, three +anonymous,o1|root:bar:slist:4=string, four +anonymous,o2|root:bar:slist:5=string, three o2 +anonymous,o2|root:bar:slist:6=string, four o2 +anonymous,global|root:bar:smap=map, java.lang.String +anonymous,global|root:bar:smap:a=string, 65 +anonymous,global|root:bar:smap:b=string, 66 +anonymous,global|root:bar:smap:c=string, 67 \ No newline at end of file diff --git a/commons/src/test/resources/org/archive/settings/path/global.get.txt b/commons/src/test/resources/org/archive/settings/path/global.get.txt new file mode 100644 index 00000000..faf3abb0 --- /dev/null +++ b/commons/src/test/resources/org/archive/settings/path/global.get.txt @@ -0,0 +1,23 @@ +global|root=map, java.lang.Object +global|root:first=object, org.archive.settings.path.Foo +global|root:primary=primary, org.archive.settings.path.Foo +global|root:second=object, org.archive.settings.path.Foo +global|root:bar=object, org.archive.settings.path.Bar +global|root:bar:foo=object, org.archive.settings.path.Foo +global|root:bar:list=list, org.archive.settings.path.Foo +global|root:bar:list:0=object, org.archive.settings.path.Foo +global|root:bar:list:1=object, org.archive.settings.path.Foo +global|root:bar:list:2=object, org.archive.settings.path.Foo +global|root:bar:map=map, org.archive.settings.path.Foo +global|root:bar:map:a=object, org.archive.settings.path.Foo +global|root:bar:map:b=object, org.archive.settings.path.Foo +global|root:bar:map:b:five=int, 50000 +global|root:bar:map:c=object, org.archive.settings.path.Foo +global|root:bar:slist=list, java.lang.String +global|root:bar:slist:0=string, zero +global|root:bar:slist:1=string, one +global|root:bar:slist:2=string, two +global|root:bar:smap=map, java.lang.String +global|root:bar:smap:a=string, 65 +global|root:bar:smap:b=string, 66 +global|root:bar:smap:c=string, 67 \ No newline at end of file diff --git a/commons/src/test/resources/org/archive/settings/path/global.resolved.txt b/commons/src/test/resources/org/archive/settings/path/global.resolved.txt new file mode 100644 index 00000000..8f79b727 --- /dev/null +++ b/commons/src/test/resources/org/archive/settings/path/global.resolved.txt @@ -0,0 +1,163 @@ +global|root=map, java.lang.Object +global|root:first=object, org.archive.settings.path.Foo +default|root:first:eight=biginteger, 8 +default|root:first:eleven=pattern, ^11$ +default|root:first:five=int, 5 +default|root:first:four=float, 4.0 +default|root:first:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:first:nine=bigdecimal, 9.0 +default|root:first:one=byte, 1 +default|root:first:seven=short, 7 +default|root:first:six=long, 6 +default|root:first:ten=string, ten +default|root:first:three=double, 3.0 +default|root:first:twelve=file, /12 +default|root:first:two=char, 2 +default|root:first:zero=boolean, false +global|root:primary=primary, org.archive.settings.path.Foo +default|root:primary:eight=biginteger, 8 +default|root:primary:eleven=pattern, ^11$ +default|root:primary:five=int, 5 +default|root:primary:four=float, 4.0 +default|root:primary:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:primary:nine=bigdecimal, 9.0 +default|root:primary:one=byte, 1 +default|root:primary:seven=short, 7 +default|root:primary:six=long, 6 +default|root:primary:ten=string, ten +default|root:primary:three=double, 3.0 +default|root:primary:twelve=file, /12 +default|root:primary:two=char, 2 +default|root:primary:zero=boolean, false +global|root:second=object, org.archive.settings.path.Foo +default|root:second:eight=biginteger, 8 +default|root:second:eleven=pattern, ^11$ +default|root:second:five=int, 5 +default|root:second:four=float, 4.0 +default|root:second:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:second:nine=bigdecimal, 9.0 +default|root:second:one=byte, 1 +default|root:second:seven=short, 7 +default|root:second:six=long, 6 +default|root:second:ten=string, ten +default|root:second:three=double, 3.0 +default|root:second:twelve=file, /12 +default|root:second:two=char, 2 +default|root:second:zero=boolean, false +global|root:bar=object, org.archive.settings.path.Bar +global|root:bar:foo=object, org.archive.settings.path.Foo +default|root:bar:foo:eight=biginteger, 8 +default|root:bar:foo:eleven=pattern, ^11$ +default|root:bar:foo:five=int, 5 +default|root:bar:foo:four=float, 4.0 +default|root:bar:foo:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:foo:nine=bigdecimal, 9.0 +default|root:bar:foo:one=byte, 1 +default|root:bar:foo:seven=short, 7 +default|root:bar:foo:six=long, 6 +default|root:bar:foo:ten=string, ten +default|root:bar:foo:three=double, 3.0 +default|root:bar:foo:twelve=file, /12 +default|root:bar:foo:two=char, 2 +default|root:bar:foo:zero=boolean, false +default|root:bar:foo-auto=auto, root:primary +global|root:bar:list=list, org.archive.settings.path.Foo +global|root:bar:list:0=object, org.archive.settings.path.Foo +default|root:bar:list:0:eight=biginteger, 8 +default|root:bar:list:0:eleven=pattern, ^11$ +default|root:bar:list:0:five=int, 5 +default|root:bar:list:0:four=float, 4.0 +default|root:bar:list:0:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:list:0:nine=bigdecimal, 9.0 +default|root:bar:list:0:one=byte, 1 +default|root:bar:list:0:seven=short, 7 +default|root:bar:list:0:six=long, 6 +default|root:bar:list:0:ten=string, ten +default|root:bar:list:0:three=double, 3.0 +default|root:bar:list:0:twelve=file, /12 +default|root:bar:list:0:two=char, 2 +default|root:bar:list:0:zero=boolean, false +global|root:bar:list:1=object, org.archive.settings.path.Foo +default|root:bar:list:1:eight=biginteger, 8 +default|root:bar:list:1:eleven=pattern, ^11$ +default|root:bar:list:1:five=int, 5 +default|root:bar:list:1:four=float, 4.0 +default|root:bar:list:1:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:list:1:nine=bigdecimal, 9.0 +default|root:bar:list:1:one=byte, 1 +default|root:bar:list:1:seven=short, 7 +default|root:bar:list:1:six=long, 6 +default|root:bar:list:1:ten=string, ten +default|root:bar:list:1:three=double, 3.0 +default|root:bar:list:1:twelve=file, /12 +default|root:bar:list:1:two=char, 2 +default|root:bar:list:1:zero=boolean, false +global|root:bar:list:2=object, org.archive.settings.path.Foo +default|root:bar:list:2:eight=biginteger, 8 +default|root:bar:list:2:eleven=pattern, ^11$ +default|root:bar:list:2:five=int, 5 +default|root:bar:list:2:four=float, 4.0 +default|root:bar:list:2:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:list:2:nine=bigdecimal, 9.0 +default|root:bar:list:2:one=byte, 1 +default|root:bar:list:2:seven=short, 7 +default|root:bar:list:2:six=long, 6 +default|root:bar:list:2:ten=string, ten +default|root:bar:list:2:three=double, 3.0 +default|root:bar:list:2:twelve=file, /12 +default|root:bar:list:2:two=char, 2 +default|root:bar:list:2:zero=boolean, false +global|root:bar:map=map, org.archive.settings.path.Foo +global|root:bar:map:a=object, org.archive.settings.path.Foo +default|root:bar:map:a:eight=biginteger, 8 +default|root:bar:map:a:eleven=pattern, ^11$ +default|root:bar:map:a:five=int, 5 +default|root:bar:map:a:four=float, 4.0 +default|root:bar:map:a:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:map:a:nine=bigdecimal, 9.0 +default|root:bar:map:a:one=byte, 1 +default|root:bar:map:a:seven=short, 7 +default|root:bar:map:a:six=long, 6 +default|root:bar:map:a:ten=string, ten +default|root:bar:map:a:three=double, 3.0 +default|root:bar:map:a:twelve=file, /12 +default|root:bar:map:a:two=char, 2 +default|root:bar:map:a:zero=boolean, false +global|root:bar:map:b=object, org.archive.settings.path.Foo +default|root:bar:map:b:eight=biginteger, 8 +default|root:bar:map:b:eleven=pattern, ^11$ +global|root:bar:map:b:five=int, 50000 +default|root:bar:map:b:four=float, 4.0 +default|root:bar:map:b:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:map:b:nine=bigdecimal, 9.0 +default|root:bar:map:b:one=byte, 1 +default|root:bar:map:b:seven=short, 7 +default|root:bar:map:b:six=long, 6 +default|root:bar:map:b:ten=string, ten +default|root:bar:map:b:three=double, 3.0 +default|root:bar:map:b:twelve=file, /12 +default|root:bar:map:b:two=char, 2 +default|root:bar:map:b:zero=boolean, false +global|root:bar:map:c=object, org.archive.settings.path.Foo +default|root:bar:map:c:eight=biginteger, 8 +default|root:bar:map:c:eleven=pattern, ^11$ +default|root:bar:map:c:five=int, 5 +default|root:bar:map:c:four=float, 4.0 +default|root:bar:map:c:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:map:c:nine=bigdecimal, 9.0 +default|root:bar:map:c:one=byte, 1 +default|root:bar:map:c:seven=short, 7 +default|root:bar:map:c:six=long, 6 +default|root:bar:map:c:ten=string, ten +default|root:bar:map:c:three=double, 3.0 +default|root:bar:map:c:twelve=file, /12 +default|root:bar:map:c:two=char, 2 +default|root:bar:map:c:zero=boolean, false +global|root:bar:slist=list, java.lang.String +global|root:bar:slist:0=string, zero +global|root:bar:slist:1=string, one +global|root:bar:slist:2=string, two +global|root:bar:smap=map, java.lang.String +global|root:bar:smap:a=string, 65 +global|root:bar:smap:b=string, 66 +global|root:bar:smap:c=string, 67 \ No newline at end of file diff --git a/commons/src/test/resources/org/archive/settings/path/o1.get.txt b/commons/src/test/resources/org/archive/settings/path/o1.get.txt new file mode 100644 index 00000000..d59ca2ef --- /dev/null +++ b/commons/src/test/resources/org/archive/settings/path/o1.get.txt @@ -0,0 +1,12 @@ +o1,global|root=map, java.lang.Object +o1|root:first:ten=string, three plus seven +o1|root:second=object, org.archive.settings.path.Foo +o1,global|root:bar:list=list, org.archive.settings.path.Foo +o1|root:bar:list:3=object, org.archive.settings.path.Foo +o1|root:bar:list:4=object, org.archive.settings.path.Foo +o1,global|root:bar:map=map, org.archive.settings.path.Foo +o1|root:bar:map:b=object, org.archive.settings.path.Baz +o1|root:bar:map:d=object, org.archive.settings.path.Foo +o1,global|root:bar:slist=list, java.lang.String +o1|root:bar:slist:3=string, three +o1|root:bar:slist:4=string, four \ No newline at end of file diff --git a/commons/src/test/resources/org/archive/settings/path/o1.resolved.txt b/commons/src/test/resources/org/archive/settings/path/o1.resolved.txt new file mode 100644 index 00000000..31a48b73 --- /dev/null +++ b/commons/src/test/resources/org/archive/settings/path/o1.resolved.txt @@ -0,0 +1,210 @@ +o1,global|root=map, java.lang.Object +global|root:first=object, org.archive.settings.path.Foo +default|root:first:eight=biginteger, 8 +default|root:first:eleven=pattern, ^11$ +default|root:first:five=int, 5 +default|root:first:four=float, 4.0 +default|root:first:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:first:nine=bigdecimal, 9.0 +default|root:first:one=byte, 1 +default|root:first:seven=short, 7 +default|root:first:six=long, 6 +o1|root:first:ten=string, three plus seven +default|root:first:three=double, 3.0 +default|root:first:twelve=file, /12 +default|root:first:two=char, 2 +default|root:first:zero=boolean, false +global|root:primary=primary, org.archive.settings.path.Foo +default|root:primary:eight=biginteger, 8 +default|root:primary:eleven=pattern, ^11$ +default|root:primary:five=int, 5 +default|root:primary:four=float, 4.0 +default|root:primary:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:primary:nine=bigdecimal, 9.0 +default|root:primary:one=byte, 1 +default|root:primary:seven=short, 7 +default|root:primary:six=long, 6 +default|root:primary:ten=string, ten +default|root:primary:three=double, 3.0 +default|root:primary:twelve=file, /12 +default|root:primary:two=char, 2 +default|root:primary:zero=boolean, false +o1|root:second=object, org.archive.settings.path.Foo +default|root:second:eight=biginteger, 8 +default|root:second:eleven=pattern, ^11$ +default|root:second:five=int, 5 +default|root:second:four=float, 4.0 +default|root:second:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:second:nine=bigdecimal, 9.0 +default|root:second:one=byte, 1 +default|root:second:seven=short, 7 +default|root:second:six=long, 6 +default|root:second:ten=string, ten +default|root:second:three=double, 3.0 +default|root:second:twelve=file, /12 +default|root:second:two=char, 2 +default|root:second:zero=boolean, false +global|root:bar=object, org.archive.settings.path.Bar +global|root:bar:foo=object, org.archive.settings.path.Foo +default|root:bar:foo:eight=biginteger, 8 +default|root:bar:foo:eleven=pattern, ^11$ +default|root:bar:foo:five=int, 5 +default|root:bar:foo:four=float, 4.0 +default|root:bar:foo:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:foo:nine=bigdecimal, 9.0 +default|root:bar:foo:one=byte, 1 +default|root:bar:foo:seven=short, 7 +default|root:bar:foo:six=long, 6 +default|root:bar:foo:ten=string, ten +default|root:bar:foo:three=double, 3.0 +default|root:bar:foo:twelve=file, /12 +default|root:bar:foo:two=char, 2 +default|root:bar:foo:zero=boolean, false +default|root:bar:foo-auto=auto, root:primary +o1,global|root:bar:list=list, org.archive.settings.path.Foo +global|root:bar:list:0=object, org.archive.settings.path.Foo +default|root:bar:list:0:eight=biginteger, 8 +default|root:bar:list:0:eleven=pattern, ^11$ +default|root:bar:list:0:five=int, 5 +default|root:bar:list:0:four=float, 4.0 +default|root:bar:list:0:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:list:0:nine=bigdecimal, 9.0 +default|root:bar:list:0:one=byte, 1 +default|root:bar:list:0:seven=short, 7 +default|root:bar:list:0:six=long, 6 +default|root:bar:list:0:ten=string, ten +default|root:bar:list:0:three=double, 3.0 +default|root:bar:list:0:twelve=file, /12 +default|root:bar:list:0:two=char, 2 +default|root:bar:list:0:zero=boolean, false +global|root:bar:list:1=object, org.archive.settings.path.Foo +default|root:bar:list:1:eight=biginteger, 8 +default|root:bar:list:1:eleven=pattern, ^11$ +default|root:bar:list:1:five=int, 5 +default|root:bar:list:1:four=float, 4.0 +default|root:bar:list:1:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:list:1:nine=bigdecimal, 9.0 +default|root:bar:list:1:one=byte, 1 +default|root:bar:list:1:seven=short, 7 +default|root:bar:list:1:six=long, 6 +default|root:bar:list:1:ten=string, ten +default|root:bar:list:1:three=double, 3.0 +default|root:bar:list:1:twelve=file, /12 +default|root:bar:list:1:two=char, 2 +default|root:bar:list:1:zero=boolean, false +global|root:bar:list:2=object, org.archive.settings.path.Foo +default|root:bar:list:2:eight=biginteger, 8 +default|root:bar:list:2:eleven=pattern, ^11$ +default|root:bar:list:2:five=int, 5 +default|root:bar:list:2:four=float, 4.0 +default|root:bar:list:2:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:list:2:nine=bigdecimal, 9.0 +default|root:bar:list:2:one=byte, 1 +default|root:bar:list:2:seven=short, 7 +default|root:bar:list:2:six=long, 6 +default|root:bar:list:2:ten=string, ten +default|root:bar:list:2:three=double, 3.0 +default|root:bar:list:2:twelve=file, /12 +default|root:bar:list:2:two=char, 2 +default|root:bar:list:2:zero=boolean, false +o1|root:bar:list:3=object, org.archive.settings.path.Foo +default|root:bar:list:3:eight=biginteger, 8 +default|root:bar:list:3:eleven=pattern, ^11$ +default|root:bar:list:3:five=int, 5 +default|root:bar:list:3:four=float, 4.0 +default|root:bar:list:3:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:list:3:nine=bigdecimal, 9.0 +default|root:bar:list:3:one=byte, 1 +default|root:bar:list:3:seven=short, 7 +default|root:bar:list:3:six=long, 6 +default|root:bar:list:3:ten=string, ten +default|root:bar:list:3:three=double, 3.0 +default|root:bar:list:3:twelve=file, /12 +default|root:bar:list:3:two=char, 2 +default|root:bar:list:3:zero=boolean, false +o1|root:bar:list:4=object, org.archive.settings.path.Foo +default|root:bar:list:4:eight=biginteger, 8 +default|root:bar:list:4:eleven=pattern, ^11$ +default|root:bar:list:4:five=int, 5 +default|root:bar:list:4:four=float, 4.0 +default|root:bar:list:4:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:list:4:nine=bigdecimal, 9.0 +default|root:bar:list:4:one=byte, 1 +default|root:bar:list:4:seven=short, 7 +default|root:bar:list:4:six=long, 6 +default|root:bar:list:4:ten=string, ten +default|root:bar:list:4:three=double, 3.0 +default|root:bar:list:4:twelve=file, /12 +default|root:bar:list:4:two=char, 2 +default|root:bar:list:4:zero=boolean, false +o1,global|root:bar:map=map, org.archive.settings.path.Foo +global|root:bar:map:a=object, org.archive.settings.path.Foo +default|root:bar:map:a:eight=biginteger, 8 +default|root:bar:map:a:eleven=pattern, ^11$ +default|root:bar:map:a:five=int, 5 +default|root:bar:map:a:four=float, 4.0 +default|root:bar:map:a:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:map:a:nine=bigdecimal, 9.0 +default|root:bar:map:a:one=byte, 1 +default|root:bar:map:a:seven=short, 7 +default|root:bar:map:a:six=long, 6 +default|root:bar:map:a:ten=string, ten +default|root:bar:map:a:three=double, 3.0 +default|root:bar:map:a:twelve=file, /12 +default|root:bar:map:a:two=char, 2 +default|root:bar:map:a:zero=boolean, false +o1|root:bar:map:b=object, org.archive.settings.path.Baz +default|root:bar:map:b:eight=biginteger, 8 +default|root:bar:map:b:eleven=pattern, ^11$ +default|root:bar:map:b:five=int, 5 +default|root:bar:map:b:four=float, 4.0 +default|root:bar:map:b:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:map:b:nine=bigdecimal, 9.0 +default|root:bar:map:b:one=byte, 1 +default|root:bar:map:b:seven=short, 7 +default|root:bar:map:b:six=long, 6 +default|root:bar:map:b:ten=string, ten +default|root:bar:map:b:three=double, 3.0 +default|root:bar:map:b:twelve=file, /12 +default|root:bar:map:b:two=char, 2 +default|root:bar:map:b:zero=boolean, false +global|root:bar:map:c=object, org.archive.settings.path.Foo +default|root:bar:map:c:eight=biginteger, 8 +default|root:bar:map:c:eleven=pattern, ^11$ +default|root:bar:map:c:five=int, 5 +default|root:bar:map:c:four=float, 4.0 +default|root:bar:map:c:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:map:c:nine=bigdecimal, 9.0 +default|root:bar:map:c:one=byte, 1 +default|root:bar:map:c:seven=short, 7 +default|root:bar:map:c:six=long, 6 +default|root:bar:map:c:ten=string, ten +default|root:bar:map:c:three=double, 3.0 +default|root:bar:map:c:twelve=file, /12 +default|root:bar:map:c:two=char, 2 +default|root:bar:map:c:zero=boolean, false +o1|root:bar:map:d=object, org.archive.settings.path.Foo +default|root:bar:map:d:eight=biginteger, 8 +default|root:bar:map:d:eleven=pattern, ^11$ +default|root:bar:map:d:five=int, 5 +default|root:bar:map:d:four=float, 4.0 +default|root:bar:map:d:moose=enum, org.archive.settings.path.Foo$Zoo-MOOSE +default|root:bar:map:d:nine=bigdecimal, 9.0 +default|root:bar:map:d:one=byte, 1 +default|root:bar:map:d:seven=short, 7 +default|root:bar:map:d:six=long, 6 +default|root:bar:map:d:ten=string, ten +default|root:bar:map:d:three=double, 3.0 +default|root:bar:map:d:twelve=file, /12 +default|root:bar:map:d:two=char, 2 +default|root:bar:map:d:zero=boolean, false +o1,global|root:bar:slist=list, java.lang.String +global|root:bar:slist:0=string, zero +global|root:bar:slist:1=string, one +global|root:bar:slist:2=string, two +o1|root:bar:slist:3=string, three +o1|root:bar:slist:4=string, four +global|root:bar:smap=map, java.lang.String +global|root:bar:smap:a=string, 65 +global|root:bar:smap:b=string, 66 +global|root:bar:smap:c=string, 67 \ No newline at end of file diff --git a/dist/HOWTO-Launch-Heritrix.txt b/dist/HOWTO-Launch-Heritrix.txt new file mode 100644 index 00000000..ff87412a --- /dev/null +++ b/dist/HOWTO-Launch-Heritrix.txt @@ -0,0 +1,302 @@ +------------------------------------------------------------------------------- +$Id$ +------------------------------------------------------------------------------- + + HOWTO: Launch Heritrix + +0.0 Contents +1.0 Introduction + 1.1 Foreground vs. Background +2.0 HOWTO: Start Heritrix with Web UI and Crawl Engine + 2.1 Troubleshooting +3.0 HOWTO: Start Heritrix with Crawl Engine only +4.0 HOWTO: Start Heritrix with Web UI only +5.0 HOWTO: Shutdown Heritrix + 5.1 Web UI + 5.2 Command-line: Foreground + 5.3 Command-line: Background + +=============================================================================== +1.0 Introduction +=============================================================================== + +The most up-to-date version of this document can be found at: + + http://webteam.archive.org/confluence/display/Heritrix/HOWTO+Launch+Heritrix + +The purpose of this HOWTO is to help Heritrix users launch Heritrix in +the desired mode with the desired features enabled. + +Heritrix can be run in the foreground as a regular application with +log messages, errors and other output going to the terminal; or it can +be run in the background with output going to a log file. + +In addition, Heritrix can be run with different combinations of +high-level features turned on or off. For example: + + Web UI + Crawl Engine + + This is suitable for a single-machine scenario where the user + wishes to use the Web UI to configure a crawl and then run the + crawl job, all on the same machine. + + Crawl Engine only + + In situation, a user wishes to run just the crawl engine, but + without the Web UI. This is common in larger and more advanced + crawls where a crawl job is split up among multiple engines + running on multiple machines. + + In this scenario, the crawl engines are typically controlled via + the JMX interface, or from a remote Web UI running on a different + machine. + + Web UI only + + The Web UI can be used to connect to remote crawl engines, running + on different machines. In this case, the user starts up Heritrix + with just he Web UI and no crawl engine running locally, then in + the Web UI, connects to remote crawl engines and controls them + remotely. + +The following sections are aimed at helping users launch Heritrix for +these common scenarios. + +1.1 Foreground vs. Background +----------------------------- +There are two Heritrix launcher scripts in the './bin' directory: + + heritrix + foreground_heritrix + +These scripts are essentially the same, except that +'foreground_heritrix' launches Heritrix in the foreground and +'heritrix' launches Heritrix in the background. Otherwise, they +are the same. + +Since Heritrix is typically intended to be a long-running server, by +default it is launched in the background and its output is sent to +"heritrix_out.log". This way, after Heritrix is launched, the user +can logout of the terminal session and Heritrix will remain running +with the output captured to the log. + +However, some users may wish to launch Heritrix in the foreground, +allowing the output to go to the terminal. This is easily achieved +by either + + o using 'foreground_heritrix' rather than 'heritrix', with + the same command-line options + + o set the environment variable 'FOREGROUND' to a non-empty value, + then launch with 'heritrix' as normal. + +We recommend running Heritrix in the background. All of the examples +and tutorials below launch Heritrix is in the background. + +=============================================================================== +2.0 HOWTO: Start Heritrix with Web UI and Crawl Engine +=============================================================================== + + $ ./bin/heritrix -a admin + +This will startup Heritrix with the Crawl Engine and the Web UI +both enabled. The + + -a admin + +command-line argument sets the Web UI's password to "admin". + +Upon successful launch of Heritrix, you should see something like the +following on your terminal + + heritrix-2.0.0 $ ./bin/heritrix -a admin + WARNING: $HERITRIX_HOME/conf/jmxremote.password not found. + WARNING: Disabling remote JMX. + Tue Feb 5 10:47:00 PST 2008 Starting heritrix.... + No JNDI context. + Engine registered at org.archive.crawler:instance=11985823,jmxport=-1,name=Engine,type=org.archive.crawler.framework.Engine,host=localhost + Web UI listening on localhost:8080. + +Once control is returned to your terminal, Heritrix will be running in +the background and all logging and error messages are sent to + + ./heritrix_out.log + +Now that Heritrix is up and running, you can visit + + http://localhost:8080/ + +in your browser to login to the Heritrix Web UI using the "admin" +password you specified on the command line. + +Also, there is a thorough step-by-step tutorial using the Web UI to +configure and run a sample crawl on the Heritrix wiki + + http://webteam.archive.org/confluence/display/Heritrix/2.0+Tutorial + +2.1 Troubleshooting +------------------- +The most common errors that occur when launching Heritrix with the Web +UI enabled are: + + o Forgetting the -a [password] command-line argument + + If this it omitted, then Heritrix will be launched without the Web + UI enabled. You won't be able to access it via the browser. + + The simplest solution is to kill Heritrix and re-launch with the + "-a" argument. + + o Launching more than one instance of Heritrix + + Once Heritrix is launched with the Web UI enabled, it will listen + on port 8080 (or another port if given -p). Only one application + at a time can listen on a port. + + If you launch a second Heritrix with the Web UI and the same port, + you will see an error message at start-up like + + java.net.BindException: Address already in use + [stack trace] + + See section 5.0 for instructions on how to shutdown Heritrix. + + +=============================================================================== +3.0 HOWTO: Start Heritrix with Crawl Engine only +=============================================================================== + +If Heritrix is launched without the Web UI, then the JMX interface +must be properly configured in order for the crawler engine to be +controlled remotely. + +The easiest way to do this is to copy and edit the JMX password +template file. In addition, as required by JMX, the password file +must be readable only by the owner. For example, + + heritrix-2.0.0 $ cd conf + conf $ cp jmxremote.password.template jmxremote.password + conf $ chmod 600 jmxremote.password + conf $ cd .. + +then edit "jmxremote.password" in your favorite editor and edit +the lines: + + #monitorRole [password goes here] + #controlRole [password goes here] + +by removing the "#" character at the start of each and setting the +passwords to your liking, such as: + + monitorRole somePassword + controlRole someOtherPassword + +Then launch Heritrix with the "-n" option to disable the Web UI. + + heritrix-2.0.0 $ ./bin/heritrix -n + Tue Feb 5 12:53:29 PST 2008 Starting heritrix + No JNDI context. + Engine registered at org.archive.crawler:instance=28290629,jmxport=8849,name=Engine,type=org.archive.crawler.framework.Engine,host=localhost + Not running web UI. + +Once control is returned to your terminal, Heritrix will be running in +the background and all logging and error messages are sent to + + ./heritrix_out.log + +Now that the crawl engine is running and the JMX interface is enabled, +you can connect to it via a JMX client such as 'jconsole' or from a +remote Heritrix Web UI running on another machine. + +3.1 Troubleshooting +------------------- +The most common problems with running Heritrix with the JMX interface +enabled are + + o The conf/jmxremote.password file doesn't exist. + + You'll see a Heritrix startup message like the following: + + WARNING: $HERITRIX_HOME/conf/jmxremote.password not found. + + Copy the jmxremote.password.template file to jmxremote.password and + edit it as described above. + + o The conf/jmxremote.password file doesn't have the proper permissions. + + You'll see a Heritrix startup message like the following: + + Error: Password file read access must be restricted: conf/jmxremote.password + + Change the permissions on the file so that it can only be read by + the owner. + + +=============================================================================== +4.0 HOWTO: Start Heritrix with Web UI only +=============================================================================== + +To launch Heritrix with the Web UI only, use the same command-line +argument to enable the Web UI as described in 2.0, and add "-u" to +disable the crawl engine. For example, + + $ ./bin/heritrix -a admin -u + WARNING: $HERITRIX_HOME/conf/jmxremote.password not found. + WARNING: Disabling remote JMX. + Tue Feb 5 12:26:52 PST 2008 Starting heritrix.. + Not running crawl engine. + Web UI listening on localhost:8080. + +Notice in the startup message that the crawl engine is not running. + +Once control is returned to your terminal, Heritrix will be running in +the background and all logging and error messages are sent to + + ./heritrix_out.log + +You can login to the Web UI at + + http://localhost:8080/ + +and then connect to remote crawl engines via the Web UI. + + +=============================================================================== +5.0 HOWTO: Shutdown Heritrix +=============================================================================== + +Depending on how Heritrix is launched it can be shutdown in various +ways. + +5.1 Web UI +---------- +The Web UI can be shutdown via the Web UI itself. From the Web UI +home page, look for the link "Terminate the Web UI..." near the bottom +of the page. This will shutdown the Web UI and the Java Virtual +Machine (JVM) in which it runs. + +This means that if Heritrix is started with both the crawl engine and +Web UI in a single JVM, then the entire JVM is shutdown, including the +Web UI and the crawl engine. + +If, however, the Web UI was launched by itself and was used to connect +to remote crawl engines, only the Web UI will be shutdown. The remote +crawl engines are unaffected. + +5.2 Command-line: Foreground +---------------------------- +If Heritrix is started in the foreground, then it can be terminated by +pressing [Ctrl]-C in the terminal window, thus sending the Java +application a termination signal. + +5.3 Command-line: Background +---------------------------- +If Heritrix is started in the background (as is the default), then it +can be easily terminated with the 'kill' command. The process ID for +Heritrix is stored in the file "heritrix.pid" when Heritrix is +successfully launched in the background. For example + + $ cat heritrix.pid + 24593 + $ kill 24593 + diff --git a/dist/LICENSE.txt b/dist/LICENSE.txt new file mode 100644 index 00000000..8b3404d7 --- /dev/null +++ b/dist/LICENSE.txt @@ -0,0 +1,516 @@ +Heritrix is free software; you can redistribute it and/or modify it +under the terms of the GNU Lesser Public license (LGPL) reproduced +below. + +Individual source code files may be subject to or offered under other +licenses, as indicated therein. + +Heritrix is distributed with the libraries it depends upon. The +libraries can be found under the 'lib' directory, and are used under +the terms of their respective licenses, which are included alongside +the libraries in the 'lib' directory. + +-- + + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! diff --git a/dist/README.txt b/dist/README.txt new file mode 100644 index 00000000..3c295e00 --- /dev/null +++ b/dist/README.txt @@ -0,0 +1,66 @@ +------------------------------------------------------------------------------- +$Id$ +------------------------------------------------------------------------------- +0.0 Contents + +1.0 Introduction +2.0 Online Reference +3.0 Getting Started Tips +4.0 License + +1.0 Introduction + +Heritrix is the Internet Archive's open-source, extensible, web-scale, +archival-quality web crawler project. Heritrix (sometimes spelled +heretrix, or misspelled or missaid as heratrix/heritix/heretix/heratix) +is an archaic word for heiress (woman who inherits). Our crawler seeks +to collect and preserve the digital artifacts of our culture for the +benefit of future researchers and generations. + +2.0 Online Reference + +The most up-to-date information about Heritrix is on the project wiki: + + http://webteam.archive.org/confluence/display/Heritrix/2.0.0 + +3.0 Getting Started Tips + +The shell script 'heritrix' in the 'bin' directory is usually +sufficient to launch Heritrix. You must use the '-a' launch flag to set +an authentication password on the web user interface. You must use the +'-b' launch flag if you want the web user interface to accept non-local +connections. + +The bundled job profiles are good starting points for designing your +own crawl configurations. However, they each require several changes +before they will work for crawling: + +- You must configure an 'operator-contact-url' on the job's global + settings sheet. This URL will be added to the 'User-Agent' included + on your crawl's outbound traffic, and should be an HTTP URL supplying + information about the purpose of your crawl and containing contact + information if visited sites need to report problems. +- You must supply one or more 'seed' URLs to serve as crawl starting + points. + +4.0 License + +Heritrix is free software; you can redistribute it and/or modify it +under the terms of the GNU Lesser Public License as published by the +Free Software Foundation; either version 2.1 of the License, or any +later version. + +Heritrix is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser Public License for more details. + +You should have received a copy of the GNU Lesser Public License +along with Heritrix (See LICENSE.txt); if not, write to the Free +Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +02111-1307 USA + +Heritrix includes a variety of other open source libraries under the +terms of their respective licenses. Please consult those individual +licenses to learn whether the libraries are usable and redistributable +in contexts other than the Heritrix distribution. diff --git a/dist/pom.xml b/dist/pom.xml new file mode 100644 index 00000000..d3eb180f --- /dev/null +++ b/dist/pom.xml @@ -0,0 +1,149 @@ + + + + org.archive + heritrix + 3.0.0-SNAPSHOT + + 4.0.0 + org.archive.heritrix + heritrix + pom + Heritrix (for distribution) + + + + true + daily + warn + + + true + never + fail + + agilejava + agilejava.com + http://www.agilejava.com/maven + default + + + + + + + true + daily + warn + + + true + never + fail + + internetarchive + Internet Archive Maven Repository + http://builds.archive.org:8080/maven2 + default + + + + + true + daily + warn + + + true + never + fail + + agilejava + agilejava.com + http://www.agilejava.com/maven + default + + + + + + org.archive.heritrix + engine + 3.0.0-SNAPSHOT + compile + + + tomcat + jasper-compiler + 5.5.15 + compile + + + tomcat + jasper-runtime + 5.5.15 + compile + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.5 + 1.5 + + + + + testCompile + + test-compile + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + true + + + + + test + + test + + + + + + maven-assembly-plugin + + + + src/main/assembly/dist.xml + + + src/main/assembly/src.xml + + + + + + job.assembly.package + package + + attached + + + + + + + + diff --git a/dist/src/main/assembly/dist.xml b/dist/src/main/assembly/dist.xml new file mode 100644 index 00000000..ff6ce131 --- /dev/null +++ b/dist/src/main/assembly/dist.xml @@ -0,0 +1,51 @@ + + dist + + tar.gz + zip + + true + + + /lib + + + + + + . + / + + README.txt + LICENSE.txt + HOWTO-Launch-Heritrix.txt + + + + src/main/licenses + /lib + + + target/site + /docs + + + src/main/bin + /bin + 0755 + + + src/main/extras + /extras + + + src/main/conf + /conf + + + src/main/conf/jobs + /jobs + + + + diff --git a/dist/src/main/assembly/src.xml b/dist/src/main/assembly/src.xml new file mode 100644 index 00000000..b77e18cc --- /dev/null +++ b/dist/src/main/assembly/src.xml @@ -0,0 +1,19 @@ + + src + + tar.gz + zip + + true + + + + .. + /project + + **/target/** + + + + + diff --git a/dist/src/main/bin/arcreader b/dist/src/main/bin/arcreader new file mode 100755 index 00000000..f8f6f5b3 --- /dev/null +++ b/dist/src/main/bin/arcreader @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +## +## This script runs the arcreader main. +## +## Optional environment variables +## +## JAVA_HOME Point at a JDK install to use. +## +## HERITRIX_HOME Pointer to your heritrix install. If not present, we +## make an educated guess based of position relative to this +## script. +## +## JAVA_OPTS Java runtime options. +PRG="$0" +while [ -h "$PRG" ]; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '.*/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`/"$link" + fi +done +PRGDIR=`dirname "$PRG"` + +# Set HERITRIX_HOME. +if [ -z "$HERITRIX_HOME" ] +then + HERITRIX_HOME=`cd "$PRGDIR/.." ; pwd` +fi + +FOREGROUND='true' CLASS_MAIN='org.archive.io.arc.ARCReader' JMX_OFF='off' \ + $HERITRIX_HOME/bin/heritrix $@ diff --git a/dist/src/main/bin/arcreader.cmd b/dist/src/main/bin/arcreader.cmd new file mode 100644 index 00000000..218aab1b --- /dev/null +++ b/dist/src/main/bin/arcreader.cmd @@ -0,0 +1,35 @@ +:: This is the Windows version of the extractor shell script +:: Caveats, see heritrix.cmd +:: +:: This script runs the arcreader main. +:: +:: Optional environment variables +:: +:: JAVA_HOME Point at a JDK install to use. +:: +:: HERITRIX_HOME Pointer to your heritrix install. If not present, we +:: make an educated guess based of position relative to this +:: script. +:: +:: JAVA_OPTS Java runtime options. +@echo off + +set PRGDIR=%~p0 + +if "%PRGDIR%"=="~p0" ( + cmd /E:ON /F:ON /V:ON /c %0 %1 %2 %3 %4 %5 %6 %7 %8 %9 + goto:eof +) + +:: unset JMX_OFF afterwards if it wasn't set before +:: Won't work if script is aborted with Ctrl+C... +if not defined JMX_OFF set UNSET_JMX_OFF=true +set JMX_OFF=off +set CLASS_MAIN=org.archive.io.arc.ARCReader +call "%PRGDIR%\foreground_heritrix.cmd" %* +set CLASS_MAIN= +if not defined UNSET_JMX_OFF goto:eof +set JMX_OFF= +set UNSET_JMX_OFF= + +:eof \ No newline at end of file diff --git a/dist/src/main/bin/dependencies.xsl b/dist/src/main/bin/dependencies.xsl new file mode 100644 index 00000000..45841556 --- /dev/null +++ b/dist/src/main/bin/dependencies.xsl @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + : + + + + + + + diff --git a/dist/src/main/bin/extractor b/dist/src/main/bin/extractor new file mode 100755 index 00000000..d94f39c8 --- /dev/null +++ b/dist/src/main/bin/extractor @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +## +## This script runs the org.archive.crawler.extractor.ExtractorTool main. +## Pass '--help' to get usage message. +## +## Optional environment variables +## +## JAVA_HOME Point at a JDK install to use. +## +## HERITRIX_HOME Pointer to your heritrix install. If not present, we +## make an educated guess based of position relative to this +## script. +## +## JAVA_OPTS Java runtime options. +PRG="$0" +while [ -h "$PRG" ]; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '.*/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`/"$link" + fi +done +PRGDIR=`dirname "$PRG"` + +# Set HERITRIX_HOME. +if [ -z "$HERITRIX_HOME" ] +then + HERITRIX_HOME=`cd "$PRGDIR/.." ; pwd` +fi + +FOREGROUND='true', \ +CLASS_MAIN='org.archive.crawler.extractor.ExtractorTool' \ + $HERITRIX_HOME/bin/heritrix $@ diff --git a/dist/src/main/bin/extractor.cmd b/dist/src/main/bin/extractor.cmd new file mode 100644 index 00000000..9a8c9950 --- /dev/null +++ b/dist/src/main/bin/extractor.cmd @@ -0,0 +1,29 @@ +:: This is the Windows version of the extractor shell script +:: Caveats, see heritrix.cmd +:: +:: This script runs the org.archive.crawler.extractor.ExtractorTool main. +:: Pass '--help' to get usage message. +:: +:: Optional environment variables +:: +:: JAVA_HOME Point at a JDK install to use. +:: +:: HERITRIX_HOME Pointer to your heritrix install. If not present, we +:: make an educated guess based of position relative to this +:: script. +:: +:: JAVA_OPTS Java runtime options. +@echo off + +set PRGDIR=%~p0 + +if "%PRGDIR%"=="~p0" ( + cmd /E:ON /F:ON /V:ON /c %0 %1 %2 %3 %4 %5 %6 %7 %8 %9 + goto:eof +) + +set CLASS_MAIN=org.archive.crawler.extractor.ExtractorTool +call "%PRGDIR%\foreground_heritrix.cmd" %* +set CLASS_MAIN= + +:eof \ No newline at end of file diff --git a/dist/src/main/bin/foreground_heritrix b/dist/src/main/bin/foreground_heritrix new file mode 100755 index 00000000..dc1ef9ec --- /dev/null +++ b/dist/src/main/bin/foreground_heritrix @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +## +## This script launches the heritrix crawler and keeps the process in foreground +## +## Optional environment variables +## +## JAVA_HOME Point at a JDK install to use. +## +## HERITRIX_HOME Pointer to your heritrix install. If not present, we +## make an educated guess based of position relative to this +## script. +## +## JAVA_OPTS Java runtime options. +## +## FOREGROUND Set to any value -- e.g. 'true' -- if you want to run +## heritrix in foreground (Used by build system when it runs +## selftest to see if completed successfully or not).. +## + +# Resolve links - $0 may be a softlink +PRG="$0" +while [ -h "$PRG" ]; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '.*/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`/"$link" + fi +done +PRGDIR=`dirname "$PRG"` + +# Set HERITRIX_HOME. +if [ -z "$HERITRIX_HOME" ] +then + HERITRIX_HOME=`cd "$PRGDIR/.." ; pwd` +fi + +FOREGROUND='true' /bin/sh $HERITRIX_HOME/bin/heritrix $@ diff --git a/dist/src/main/bin/foreground_heritrix.cmd b/dist/src/main/bin/foreground_heritrix.cmd new file mode 100644 index 00000000..d21eeee2 --- /dev/null +++ b/dist/src/main/bin/foreground_heritrix.cmd @@ -0,0 +1,40 @@ +:: This is the windows version of the foreground_heritrix shell script +:: The only difference to an invokation with "heritrix.cmd" is that no extra +:: (minimized) console window is created... +:: Caveats, see heritrix.cmd +:: +:: This script launches the heritrix crawler and keeps the window in foreground +:: +:: Optional environment variables +:: +:: JAVA_HOME Point at a JDK install to use. +:: +:: HERITRIX_HOME Pointer to your heritrix install. If not present, we +:: make an educated guess based of position relative to this +:: script. +:: +:: JAVA_OPTS Java runtime options. +:: +:: FOREGROUND Set to any value -- e.g. 'true' -- if you want to run +:: heritrix in foreground (Used by build system when it runs +:: selftest to see if completed successfully or not).. +:: +@echo off + +set PRGDIR=%~p0 + +if "%PRGDIR%"=="~p0" ( + cmd /E:ON /F:ON /V:ON /c %0 %1 %2 %3 %4 %5 %6 %7 %8 %9 + goto:eof +) + +:: unset FOREGROUND afterwards if it wasn't set before +:: Won't work if script is aborted with Ctrl+C... +if not defined FOREGROUND set UNSET_FOREGROUND=true +set FOREGROUND=true +call "%PRGDIR%\heritrix.cmd" %* +if not defined UNSET_FOREGROUND goto:eof +set FOREGROUND= +set UNSET_FOREGROUND= + +:eof \ No newline at end of file diff --git a/dist/src/main/bin/heritrix b/dist/src/main/bin/heritrix new file mode 100644 index 00000000..b5a2644b --- /dev/null +++ b/dist/src/main/bin/heritrix @@ -0,0 +1,176 @@ +#!/usr/bin/env bash +## +## This script launches the heritrix crawler. +## +## Optional environment variables +## +## JAVA_HOME Point at a JDK install to use. +## +## HERITRIX_HOME Pointer to your heritrix install. If not present, we +## make an educated guess based of position relative to this +## script. +## +## HERITRIX_OUT Pathname to the Heritrix log file written when run in +## daemon mode. +## Default setting is $HERITRIX_HOME/heritrix_out.log +## +## JAVA_OPTS Java runtime options. Default setting is '-Xmx256m'. +## +## FOREGROUND Set to any value -- e.g. 'true' -- if you want to run +## heritrix in foreground (Used by build system when it runs +## selftest to see if completed successfully or not). +## + +# Resolve links - $0 may be a softlink +PRG="$0" +while [ -h "$PRG" ]; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '.*/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`/"$link" + fi +done +PRGDIR=`dirname "$PRG"` + +# Read local heritrix properties if any. +if [ -f $HOME/.heritrixrc ] +then + . $HOME/.heritrixrc +fi + +# Set HERITRIX_HOME. +if [ -z "$HERITRIX_HOME" ] +then + HERITRIX_HOME=`cd "$PRGDIR/.." ; pwd` +fi + +cd $HERITRIX_HOME + +# Find JAVA_HOME. +if [ -z "$JAVA_HOME" ] +then + JAVA=`which java` + if [ -z "$JAVA" ] + then + echo "Cannot find JAVA. Please set JAVA_HOME or your PATH." + exit 1 + fi + JAVA_BINDIR=`dirname $JAVA` + JAVA_HOME=$JAVA_BINDIR/.. +fi + +if [ -z "$JAVACMD" ] +then + # It may be defined in env - including flags!! + # See '[ 1482761 ] BDB Adler32 gc-lock OOME risk' for why we include the + # 'je.disable.java.adler32'. + JAVACMD="$JAVA_HOME/bin/java -Dje.disable.java.adler32=true" +fi + +# Ignore previous classpath. Build one that contains heritrix jar and content +# of the lib directory into the variable CP. +for jar in `ls $HERITRIX_HOME/lib/*.jar` +do + CP=${CP}:${jar} +done + +# cygwin path translation +if expr `uname` : 'CYGWIN*' > /dev/null; then + CP=`cygpath -p -w "$CP"` + HERITRIX_HOME=`cygpath -p -w "$HERITRIX_HOME"` +fi + +# Make sure of java opts. +if [ -z "$JAVA_OPTS" ] +then + JAVA_OPTS=" -Xmx256m" +fi + +# Main heritrix class. +if [ -z "$CLASS_MAIN" ] +then + CLASS_MAIN='org.archive.crawler.Heritrix' +fi + +# A bit of a hack to ensure that if the user puts -h/--help on the +# command line run CLASS_MAIN in the foreground so the output will go +# to the terminal. +for i in "$@" +do + if [ "$i" == "-h" ] || [ "$i" == "--help" ] + then + FOREGROUND="true" + break + fi +done + +# heritrix_dmesg.log contains startup output from the crawler main class. +# As soon as content appears in this log, this shell script prints the +# successful (or failed) startup content and moves off waiting on heritrix +# startup. This technique is done so we can show on the console startup +# messages emitted by java subsequent to the redirect of stdout and stderr. +startMessage="${HERITRIX_HOME}/heritrix_dmesg.log" + +# Remove any file that may have been left over from previous starts. +if [ -f $startMessage ] +then + rm -f $startmessage +fi +# Run heritrix as daemon. Redirect stdout and stderr to a file. +# Print start message with date, java version, java opts, ulimit, and uname. +if [ -z "$HERITRIX_OUT" ] +then + HERITRIX_OUT=${HERITRIX_HOME}/heritrix_out.log +fi +stdouterrlog=${HERITRIX_OUT} +echo "`date` Starting heritrix" >> $stdouterrlog +uname -a >> $stdouterrlog 2>&1 +${JAVACMD} ${JAVA_OPTS} -version >> $stdouterrlog 2>&1 +echo "JAVA_OPTS=${JAVA_OPTS}" >> $stdouterrlog +ulimit -a >> $stdouterrlog 2>&1 + +# If FOREGROUND is set, run heritrix in foreground. +if [ -n "$FOREGROUND" ] +then + CLASSPATH=${CP} $JAVACMD -Dheritrix.home=${HERITRIX_HOME} \ + -Djava.protocol.handler.pkgs=org.archive.net \ + -Dheritrix.out=${HERITRIX_OUT} ${JAVA_OPTS} ${JMX_OPTS} \ + $CLASS_MAIN $@ +else + CLASSPATH=${CP} nohup $JAVACMD -Dheritrix.home=${HERITRIX_HOME} \ + -Djava.protocol.handler.pkgs=org.archive.net \ + -Dheritrix.out=${HERITRIX_OUT} ${JAVA_OPTS} ${JMX_OPTS} \ + $CLASS_MAIN $@ >> ${stdouterrlog} 2>&1 & + HERITRIX_PID=$! + + # Wait for content in the heritrix_dmesg.log file. + echo -n "`date` Starting heritrix" + while true + do + sleep 1 + if [ -s $startMessage ] + then + echo + cat $startMessage + rm -f $startMessage + break + fi + kill -0 $HERITRIX_PID > /dev/null 2>&1 + if [ $? -eq 0 ] + then + echo $HERITRIX_PID > heritrix.pid + else + echo + echo "ERROR: JVM terminated without running Heritrix." + echo "This could be due to invalid JAVA_OPTS or JMX_PORT, etc." + echo "See heritrix_out.log for more details." + echo "Here are its last three lines: " + echo + tail -3 $stdouterrlog + break + fi + echo -n '.' + done +fi diff --git a/dist/src/main/bin/heritrix.cmd b/dist/src/main/bin/heritrix.cmd new file mode 100644 index 00000000..6cbe77bf --- /dev/null +++ b/dist/src/main/bin/heritrix.cmd @@ -0,0 +1,296 @@ +:: This script launches the heritrix crawler on windows. While Heritrix +:: is unsupported on windows, see 2.1.1.3 in the User Manual +:: [http://crawler.archive.org/articles/user_manual.html], this script was +:: provided by Eric Jensen as a convenience to the windows-afflicted. +:: +:: It is a direct translation of the heritrix linux wrapper script -- and +:: because windows is not supported on Heritrix, it will likely lag the unix +:: start script. +:: +:: See also: +:: https://sourceforge.net/tracker/index.php?func=detail&aid=1514538&group_id=73833&atid=539102 +:: +:: Versions: +:: +:: 2006-07-17 Original Version by Eric Jensen +:: +:: 2006-08-04 Disclaimer added by Michael Stack +:: +:: 2006-08-28 A few fixes by Max Schöfmann: +:: - command extensions and veriable expansion are automatically +:: enabled +:: - JMX configuration fixed (not the fancy "sed" stuff however) +:: - Try to set permissions of JMX password file if Heritrix +:: fails to start and JMX is enabled +:: - a few more small improvements (java detection, fake background +:: execution...) +:: - comments changed from rem to :: and file renamed to .cmd +:: (to make clear it won't work on Win 9x...) +:: +:: +:: Optional environment variables +:: +:: JAVA_HOME Point at a JDK install to use. +:: +:: HERITRIX_HOME Pointer to your heritrix install. If not present, we +:: make an educated guess based of position relative to this +:: script. +:: +:: HERITRIX_OUT Pathname to the Heritrix log file written when run in +:: daemon mode. +:: Default setting is %HERITRIX_HOME%\heritrix_out.log +:: +:: JAVA_OPTS Java runtime options. Default setting is '-Xmx256m'. +:: +:: FOREGROUND Set to any value -- e.g. 'true' -- if you want to run +:: heritrix in foreground (Used by build system when it runs +:: selftest to see if completed successfully or not). +:: +:: JMX_OPTS Default is to startup the JVM JMX administration +:: on port 8849 if the JVM is SUN JVM 1.5. This allows JMX +:: administration of Heritrix. If the JVM is other than the +:: SUN JDK 1.5, the arguments are ignored. If you do not want +:: to start the JVM JXM administration server on the SUN JDK +:: 1.5, set this variable to empty string. +:: +:: JMX_PORT Port you'd like the JVM JMX administration server to run +:: on. Default is 8849. +:: +:: JMX_OFF Set to a non-empty string to disable JMX (and JMX setup of +:: password file, etc.) +:: +@echo off + +set PRG=%0 +set PRGDIR=%~p0 +:: windows doesn't have a sleep command build-in +set SLEEP=ping 127.0.0.1 -n 2 -w 1000 + +if "%1"=="RUN" goto run +if "%1"=="BGR" goto run_in_background +:: preserve original command line arguments +if "%*"=="*" ( + :: windows separates things like --digest=false into "--digest" and "false" if using %1 %2 %3... + :: But as command extensions are enabled by default, this should be no problem for most users + echo NOTICE: Try starting your console with "cmd /E:ON" if you are experiencing + echo problems passing command line arguments to Heritrix + echo. + set HERITRIX_CMDLINE=%1 %2 %3 %4 %5 %6 %7 %8 %9 +) else ( + set HERITRIX_CMDLINE=%* +) +:: Enabling command extensions and delayed variable expansion +cmd /E:ON /F:ON /V:ON /c %PRG% RUN +goto :end + +:run +:: Read local heritrix properties if any. +:: To do this on Windows, tempor. rename .heritrixrc to heritrixrc.cmd +:: This is of course only useful if .heritrixrc contains Windows style "set VAR=value" statements +set RC_PATH=%HOMEPATH% +if "%RC_PATH%"=="\" set RC_PATH=\. +if defined HOMEDRIVE set RC_PATH=%HOMEDRIVE%!RC_PATH! +if exist "!RC_PATH!\.heritrixrc" ( + ren "!RC_PATH!\.heritrixrc" heritrixrc.cmd + call "!RC_PATH!\heritrixrc.cmd" + ren "!RC_PATH!\heritrixrc.cmd" .heritrixrc +) +set RC_PATH= + +:: Set HERITRIX_HOME. +if defined HERITRIX_HOME goto find_java +set HERITRIX_HOME=%PRGDIR:~0,-4% +if "%PRGDIR:~-1%"=="\" set HERITRIX_HOME=%PRGDIR:~0,-5% + +:: Find JAVA_HOME or java if JAVACMD is not defined. +:find_java +if defined JAVACMD goto java_found +if defined JAVA_HOME goto set_javacmd + +:: Try to find java if neither JAVACMD nor JAVA_HOME is set: +java -version >nul 2>&1 +:: 9009 means "command not found" +if errorlevel 9009 goto no_java_home +:: something else is wrong with executing java +if errorlevel 1 goto no_java_home + +:: java seems to be in PATH +set JAVACMD=java -Dje.disable.java.adler32=true +:set_javacmd +if not defined JAVACMD set JAVACMD="%JAVA_HOME%\bin\java" -Dje.disable.java.adler32=true +:: It may be defined in env - including flags!! +:: See '[ 1482761 ] BDB Adler32 gc-lock OOME risk' for why we include the +:: 'je.disable.java.adler32'. +:java_found + +:: Ignore previous classpath. Build one that contains heritrix jar and content +:: of the lib directory into the variable CP. +set CP= +set OLD_CLASSPATH=%CLASSPATH% +for %%j in ("%HERITRIX_HOME%\lib\*.jar" "%HERITRIX_HOME%\*.jar") do set CP=!CP!;%%j +set CLASSPATH=!CP! + +:: DONT cygwin path translation +:: if expr `uname` : 'CYGWIN*' > /dev/null; then +:: CP=`cygpath -p -w "%CP"` +:: HERITRIX_HOME=`cygpath -p -w "%HERITRIX_HOME"` +:: fi + +:: Make sure of java opts. +if not defined JAVA_OPTS set JAVA_OPTS= -Xmx256m + +:: Setting environment vars in nested IFs is error prone, thus using GOTOs +if not defined JMX_OFF goto configure_jmx +goto jmx_configured + +:configure_jmx +if not defined JMX_PORT set JMX_PORT=8849 +if not defined JMX_OPTS set JMX_OPTS=-Dcom.sun.management.jmxremote.port=%JMX_PORT% -Dcom.sun.management.jmxremote.ssl=false "-Dcom.sun.management.jmxremote.password.file=%HERITRIX_HOME%\jmxremote.password" + +:: DONT Copy into place a jmxremote password file that uses the heritrix password +:: interpolated (First need to find the current password if one supplied on +:: command-line, else use whats in heritrix.properties as default). +:: Need to make it so its only readable by user else jconsole won't use it. +:: JMX_PASSWORD=`echo "%@" |sed -n -e 's/.*--admin=[^:]*:\([^ ]*\).*/\1/p' -e 's/.*-a *[^:]*:\([^ ]*\).*/\1/p'` +:: if [ -z "%JMX_PASSWORD" ] +:: then +:: JMX_PASSWORD=`sed -n -e 's/heritrix.cmdline.admin[ ]*=[^:]*:\(.*\)/\1/p' \ +:: %{HERITRIX_HOME}\conf\heritrix.properties` +:: fi +:: JMX_PWORD_FILE="%{HERITRIX_HOME}\jmxremote.password" +:: if [ -f "%{JMX_PWORD_FILE}" ] +:: then +:: rm -f "%{JMX_PWORD_FILE}" +:: fi +:: sed -e "s/@PASSWORD@/%{JMX_PASSWORD}/" \ +:: "%{HERITRIX_HOME}\conf\jmxremote.password.template" > "%{JMX_PWORD_FILE}" +:: chmod 600 "%{JMX_PWORD_FILE}" + +:jmx_configured + +:: Main heritrix class. +if not defined CLASS_MAIN set CLASS_MAIN=org.archive.crawler.Heritrix + +:: heritrix_dmesg.log contains startup output from the crawler main class. +:: As soon as content appears in this log, this shell script prints the +:: successful (or failed) startup content and moves off waiting on heritrix +:: startup. This technique is done so we can show on the console startup +:: messages emitted by java subsequent to the redirect of stdout and stderr. +set startMessage=%HERITRIX_HOME%\heritrix_dmesg.log + +:: Remove any file that may have been left over from previous starts. +if exist "%startMessage%" del "%startmessage%" +if exist "%HERITRIX_HOME%\jmx_permissions_broken" del "%HERITRIX_HOME%\jmx_permissions_broken" + +:: Run heritrix as daemon. Redirect stdout and stderr to a file. +:: Print start message with date, java version, java opts, ulimit, and uname. +if not defined HERITRIX_OUT set HERITRIX_OUT=%HERITRIX_HOME%\heritrix_out.log + +set stdouterrlog=%HERITRIX_OUT% +echo %DATE% %TIME% Starting heritrix >>"%stdouterrlog%" +:: uname -a >> %stdouterrlog% +%JAVACMD% %JAVA_OPTS% -version >>"%stdouterrlog%" 2>&1 +echo JAVA_OPTS=%JAVA_OPTS% >>"%stdouterrlog%" +:: ulimit -a >> %stdouterrlog 2>&1 + +:: DONT If FOREGROUND is set, run heritrix in foreground. +:: if defined FOREGROUND +:start_heritrix +if not defined FOREGROUND goto run_in_background +%JAVACMD% "-Dheritrix.home=%HERITRIX_HOME%" -Djava.protocol.handler.pkgs=org.archive.net "-Dheritrix.out=%HERITRIX_OUT%" %JAVA_OPTS% %JMX_OPTS% %CLASS_MAIN% %HERITRIX_CMDLINE% +:: errorlevel 130 if aborted with Ctrl+c (at least my sun jvm 1.5_07...) +if errorlevel 130 goto :end +if errorlevel 1 goto fix_jmx_permissions +goto :end + +:run_in_background +if not "%1"=="BGR" ( + start /MIN cmd /E:ON /F:ON /V:ON /c %PRG% BGR + goto wait_for_log_file +) else ( + title Heritrix + :: adding ">>%stdouterrlog% 2>&1" causes an access denied error as heritrix writes also to this file + %JAVACMD% "-Dheritrix.home=%HERITRIX_HOME%" -Djava.protocol.handler.pkgs=org.archive.net "-Dheritrix.out=%HERITRIX_OUT%" %JAVA_OPTS% %JMX_OPTS% %CLASS_MAIN% %HERITRIX_CMDLINE% + if errorlevel 130 goto :end + if errorlevel 1 echo.!ERRORLEVEL! >"%HERITRIX_HOME%\jmx_permissions_broken" + pause + ) +goto :end + +:wait_for_log_file +SET HERITRIX_COUNTER= +echo WARNING: It's currently not possible to run Heritrix in background +echo on Windows. It was just started minimized in a new Window +echo and will be shut down as soon as you log off. +echo. +echo %DATE% %TIME% Starting heritrix +:print_logfile +%SLEEP%>nul +if exist "%HERITRIX_HOME%\jmx_permissions_broken" ( + del "%HERITRIX_HOME%\jmx_permissions_broken" + goto fix_jmx_permissions +) +if exist "%startMessage%" ( + %SLEEP%>nul + type "%startMessage%" + :: can happen when heritrix writes to the file at the same time + if errorlevel 1 goto print_logfile + goto delete_logfile +) +:: keep trying for 30 more seconds +if "!HERITRIX_COUNTER!"==".............................." goto start_may_failed +set HERITRIX_COUNTER=.!HERITRIX_COUNTER! +echo . +goto print_logfile + +:delete_logfile +set HERITRIX_COUNTER= +%SLEEP%>nul +%SLEEP%>nul +del "%startMessage%" >nul 2>&1 +:: del doesn't set the ERRORLEVEL var if unsuccessful, so we can't try again +goto :end + +:fix_jmx_permissions +if not "%CLASS_MAIN%"=="org.archive.crawler.Heritrix" goto :start_may_failed +if defined PERMISSIONS_FIXED goto fix_jmx_permission_failed +echo. +echo Heritrix failed to start properly. Possible causes: +echo. +echo - another programm uses the port for the web inferface (8080 default) +echo (e.g. another Heritrix instance) +if defined JMX_OFF goto :end +echo - permissions problem with the JMX password file. +echo. +set /P FIXIT=Do you want to try to fix the permissions (Y/N)? +if /I "%FIXIT:~0,1%"=="n" goto :end +cacls "%HERITRIX_HOME%\jmxremote.password" /P %USERNAME%:R +if errorlevel 1 goto fix_jmx_permission_failed +set PERMISSIONS_FIXED=true +set /P RESTART=Restart Heritrix (Y/N)? +if /I "%RESTART:~0,1%"=="y" goto start_heritrix +goto :end + +:fix_jmx_permission_failed +set PERMISSIONS_FIXED= +echo Either fixing the permissions failed or there was another problem +goto :end + +:start_may_failed +set HERITRIX_COUNTER= +echo Starting Heritrix seems to have failed +goto :end + +:no_java_home +echo Please define either JAVA_HOME or JAVACMD or make sure java.exe is in PATH +goto :end + +:: needed if initially called without command extensions +:end +:: do some cleanup +set HERITRIX_CMDLINE= +if defined OLD_CLASSPATH set CLASSPATH=%OLD_CLASSPATH% +set CP= +set SLEEP= +set PRGDIR= +set PRG= diff --git a/dist/src/main/bin/hoppath.pl b/dist/src/main/bin/hoppath.pl new file mode 100644 index 00000000..7079f00b --- /dev/null +++ b/dist/src/main/bin/hoppath.pl @@ -0,0 +1,124 @@ +#!/usr/bin/env perl + +# Script that parses crawl.log and prints out path to passed in URI. +# This script works by first sorting the crawl.log to a file named +# flattened.crawl.log which it writes to current directory. It then runs +# its queries against this file (If the crawl.log is changed, you'll need +# to remove the cache flattened_crawl.log file). +# +# ASSUMPTIONS: +# + This script was tested with perl 5.8 on debian. +# + Script expects unix 'sort' in path, and probably a linux sort at that +# (Takes a '-k' flag). Adjust the '$SORT' variable below if your sort acts +# differently or is in an alternate location. +# +# $Id$ +# +use strict; + +# External dependency. Change this variable to point at your local 'sort' +# install. +my $SORT = "sort -k4,4"; + +# Usage message. +my $USAGE = < $sortedLogFile") + or die "Couldn't open filehandle to $sortedLogFile: $!.\n"; + print STDOUT "Sorting crawl log file and saving to $sortedLogFile.\n" . + "May take a few minutes (This is only done once!).\n"; + my $line; + while () { + # Collapse field spaces. + tr/ //s; + print FLATLOG; + } + close(FH); + close(FLATLOG); + } + return $sortedLogFile; +} + +# Search in cached flattened log file for requested url prefix +sub search { + my ($sortedLogFile, $query) = @_; + my @stack; + my $exactMatch = 0; + open (FLATLOG, $sortedLogFile) or die "Failed open of $sortedLogFile: $!\n"; + LINES: while () { + # Split the line in to constituent parts. + chomp; + my @parts = split(/ /, $_); + my $uri = $parts[3]; + if ($exactMatch? $uri eq $query: $uri =~ m|$query|) { + # Get last path character + my $path = $parts[4]; + my $pathchar = $path? (split(//, $path))[-1]: ''; + # Push url onto stack. + push @stack, "$parts[0] $pathchar $uri\n"; + # Rewind and search for this url's referrer. + $query = $parts[5]; + if (not $query) { + # If no referrer, we're done. + last LINES; + } + $exactMatch = 1; + seek(FLATLOG, 0, 0) or die "Failed rewind: $!.\n"; + } + } + + # Print out results. + if (not @stack) { + print STDOUT "URL prefix not found: $URLPREFIX.\n"; + exit 0; + } else { + my $spaces = ''; + while (@stack) { + my @parts = split(/ /, pop(@stack)); + my $line = formatDate($parts[0]) . $spaces; + for (my $i = 1; $i < scalar @parts; $i++) { + $line .= ' ' . $parts[$i]; + } + print STDOUT "$line"; + $spaces .= " "; + } + } +} + +# Date formatting. +sub formatDate { + my @p = split(//, shift); + return "$p[0]$p[1]$p[2]$p[3]-$p[4]$p[5]-$p[6]$p[7]-$p[8]$p[9]-" . + "$p[10]$p[11]-$p[12]$p[14]"; +} diff --git a/dist/src/main/bin/htmlextractor b/dist/src/main/bin/htmlextractor new file mode 100755 index 00000000..30d6ecf5 --- /dev/null +++ b/dist/src/main/bin/htmlextractor @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +## +## This script runs the org.archive.crawler.extractor.ExtractorHTMLTest main. +## +## Optional environment variables +## +## JAVA_HOME Point at a JDK install to use. +## +## HERITRIX_HOME Pointer to your heritrix install. If not present, we +## make an educated guess based of position relative to this +## script. +## +## JAVA_OPTS Java runtime options. +PRG="$0" +while [ -h "$PRG" ]; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '.*/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`/"$link" + fi +done +PRGDIR=`dirname "$PRG"` + +# Set HERITRIX_HOME. +if [ -z "$HERITRIX_HOME" ] +then + HERITRIX_HOME=`cd "$PRGDIR/.." ; pwd` +fi + +FOREGROUND='true', \ +CLASS_MAIN='org.archive.crawler.extractor.ExtractorHTMLTest' \ + $HERITRIX_HOME/bin/heritrix $@ diff --git a/dist/src/main/bin/htmlextractor.cmd b/dist/src/main/bin/htmlextractor.cmd new file mode 100644 index 00000000..1ccaf7a0 --- /dev/null +++ b/dist/src/main/bin/htmlextractor.cmd @@ -0,0 +1,29 @@ +:: This is the Windows version of the extractor shell script +:: Caveats, see heritrix.cmd +:: +:: This script runs the org.archive.crawler.extractor.ExtractorHTMLTest main. +:: +:: Optional environment variables +:: +:: JAVA_HOME Point at a JDK install to use. +:: +:: HERITRIX_HOME Pointer to your heritrix install. If not present, we +:: make an educated guess based of position relative to this +:: script. +:: +:: JAVA_OPTS Java runtime options. +@echo off + +set PRGDIR=%~p0 + +if "%PRGDIR%"=="~p0" ( + cmd /E:ON /F:ON /V:ON /c %0 %1 %2 %3 %4 %5 %6 %7 %8 %9 + goto:eof +) + + +set CLASS_MAIN=org.archive.crawler.extractor.ExtractorHTMLTest +call "%PRGDIR%\foreground_heritrix.cmd" %* +set CLASS_MAIN= + +:eof diff --git a/dist/src/main/bin/make_reports.pl b/dist/src/main/bin/make_reports.pl new file mode 100644 index 00000000..6a0d9e33 --- /dev/null +++ b/dist/src/main/bin/make_reports.pl @@ -0,0 +1,186 @@ +#!/usr/bin/env perl + +# make_reports.pl +# +# Generates very close approximations of the standard Heritrix +# crawl reports directly from the crawl log. +# +# Useful if Heritrix ends uncleanly and is unable to create them itself +# +# requires: +# Date::Calc module +# (or just comment out the two lines with "Delta_DHMS" if time calculation isn't needed in crawl-report.txt) +# +# usage: +# make_reports.pl +# +# files created: +# ./crawl-report.txt.new +# ./hosts-report.txt.new +# ./mimetype-report.txt.new +# ./responsecode-report.txt.new +# ./seeds-report.txt.new +# ./badloglines.log (where unparseable lines go) + +use strict; +use Date::Calc qw( Delta_DHMS ); + +open( CRAWL, $ARGV[0] ) or die "Can\'t open specified crawl.log.\n$!"; +my $starttime; +my $currenttime; +my $doccount = 0; +my $hosts; +my $mimes; +my $responses; +my $seedresponses; +my $totalsize = 0; + +open( BADLINES, ">badloglines.log" ) or die "Can't create badloglines.log in current directory\n$!"; + +while ( my $line = ) { + my ( + $time, $result, $size, $url, $hoppath, $refer, + $mime, $thread, $dltime, $sha1, $sourcetag, $annot, + $overflow + ) + = split /\s+/, $line; + +# if there are more or less than 11 fields, one of the fields has a space in it, so skip parsing it + if ( ( $overflow ne "" ) || ( $annot eq "" ) ) { + print BADLINES $line; + next; + } + + # get first line's time + if ( $. == 1 ) { + $starttime = $time; + } + + # else get the time of the current line, in case this is the last line + else { + $currenttime = $time; + } + + my ($host) = ( $url =~ m|://(.+?)[:/]| ); + next unless ($host); + $seedresponses->{$url} = $result if ( $hoppath eq "-" ); + + # if not a fetching error + if ( $result > 0 ) { + $hosts->{$host}{count}++; + $hosts->{$host}{size} += $size; + + $mimes->{$mime}{count}++; + $mimes->{$mime}{size} += $size; + + $responses->{$result}{count}++; + + $totalsize += $size; + } + if ( $result >= 200 && $result <= 399 ) { + $doccount++; + } +} +my $htotalsize = sprintf("%.2f", $totalsize/1024/1024/1024); +#determine time taken for crawl +my ( $syear, $smonth, $sday, $shour, $smin, $ssec, $sms ); +( $syear, $smonth, $sday, $shour, $smin, $ssec, $sms ) = + ( $starttime =~ /(....)-(..)-(..)T(..):(..):(..).(...)Z/ ); +if (!defined $syear) { + ( $syear, $smonth, $sday, $shour, $smin, $ssec, $sms ) = + ( $starttime =~ /(....)(..)(..)(..)(..)(..)(...)/ ); +} + + +my ( $eyear, $emonth, $eday, $ehour, $emin, $esec, $ems ); +( $eyear, $emonth, $eday, $ehour, $emin, $esec, $ems ) = + ( $currenttime =~ /(....)-(..)-(..)T(..):(..):(..).(...)Z/ ); +if (!defined $eyear) { + ( $eyear, $emonth, $eday, $ehour, $emin, $esec, $ems ) = + ( $currenttime =~ /(....)(..)(..)(..)(..)(..)(...)/ ); +} +my ( $days, $hours, $minutes, $seconds ) = Delta_DHMS( + $syear, $smonth, $sday, $shour, $smin, $ssec, + $eyear, $emonth, $eday, $ehour, $emin, $esec +); + +print_crawl_report(); +print_hosts_report(); +print_mimetype_report(); +print_response_report(); +print_seeds_report(); + +sub print_crawl_report { + open( NEWCRAWL, "> crawl-report.txt.new" ) or die $!; + print NEWCRAWL "Duration Time: ", ( $days * 24 ) + $hours, + "h${minutes}m${seconds}s\n"; + print NEWCRAWL "Total Hosts Crawled: ", scalar keys %$hosts, "\n"; + print NEWCRAWL "Total Documents Crawled: ", $doccount, "\n"; + print NEWCRAWL "Total Raw Data Size in Bytes: $totalsize ($htotalsize GB)\n"; + close NEWCRAWL; +} + +sub print_hosts_report { + open( NEWHOSTS, "> hosts-report.txt.new" ) or die $!; + print NEWHOSTS "[#urls] [#bytes] [host]\n"; + + # sort from most frequent to least frequent host + my @sortedhosts = + sort { $hosts->{$b}{count} <=> $hosts->{$a}{count} } keys %$hosts; + foreach my $host (@sortedhosts) { + printf NEWHOSTS ("%s %s %s\n", + $hosts->{$host}{count}, + $hosts->{$host}{size}, + $host + ); + } +} + +sub print_mimetype_report { + open( NEWMIMES, "> mimetype-report.txt.new" ) or die $!; + print NEWMIMES "[#urls] [#bytes] [mime-types]\n"; + + # sort from most frequent to least frequent mime-type + my @sortedmimes = + sort { $mimes->{$b}{count} <=> $mimes->{$a}{count} } keys %$mimes; + foreach my $mime (@sortedmimes) { + printf NEWMIMES ("%s %s %s\n", + $mimes->{$mime}{count}, + $mimes->{$mime}{size}, + $mime + ); + } +} + +sub print_response_report { + open( NEWRESPONSES, "> responsecode-report.txt.new" ) or die $!; + print NEWRESPONSES "[rescode] [#urls]\n"; + + # sort from most frequent to least frequent response code + my @sortedresps = + sort { $responses->{$b}{count} <=> $responses->{$a}{count} } + keys %$responses; + foreach my $res (@sortedresps) { + printf NEWRESPONSES ( "%s %s\n", $res, $responses->{$res}{count} ); + } +} + +sub print_seeds_report { + open( NEWSEEDS, "> seeds-report.txt.new" ) or die $!; + + # sort urls by response code, then by url + my @sortedurls = + sort { $seedresponses->{$a} cmp $seedresponses->{$b} || $a cmp $b } + keys %$seedresponses; + print NEWSEEDS "[res-code] [status] [seeds]\n"; + foreach my $url (@sortedurls) { + # if anything but a heritrix error + if ( $seedresponses->{$url} > 0 ) { + print NEWSEEDS "$seedresponses->{$url} CRAWLED $url\n"; + } + else { + print NEWSEEDS "$seedresponses->{$url} NOTCRAWLED $url\n"; + } + } +} + diff --git a/dist/src/main/bin/manifest_bundle.pl b/dist/src/main/bin/manifest_bundle.pl new file mode 100644 index 00000000..9ee2ac8a --- /dev/null +++ b/dist/src/main/bin/manifest_bundle.pl @@ -0,0 +1,104 @@ +#!/usr/bin/env perl + +use strict; + + +my $LFLAG = "-L"; +my $CFLAG = "-C"; +my $RFLAG = "-R"; + +my $lFlagDir = "logs"; +my $cFlagDir = "configurations"; +my $rFlagDir = "reports"; +my $v = 0; + +my $tarFile; +my $tarFlags = "ch"; +my $compress = 0; +my %flags = ($LFLAG => $lFlagDir, $CFLAG => $cFlagDir, $RFLAG => $rFlagDir); + +my $crawlName = shift; +my $manifestFile = shift; + +die "Could not find $manifestFile: ($!)\n" if ( ! -f $manifestFile); + +while (@ARGV > 0) { + if ($ARGV[0] eq "-f") { + shift; + $tarFile = shift; + $tarFlags = $tarFlags . "f $tarFile"; + }elsif ($ARGV[0] eq "-z") { + shift; + $tarFlags = "z" . $tarFlags; + }elsif ($ARGV[0] eq $LFLAG) { + shift; + $lFlagDir = shift; + }elsif ($ARGV[0] eq $RFLAG){ + shift; + $rFlagDir = shift; + }elsif ($ARGV[0] eq $CFLAG){ + shift; + $cFlagDir = shift; + }elsif ($ARGV[0] eq "--help"){ + shift; + USAGE(); + }else { + # any other flag + my $key = shift; + $flags{$key} = shift; + } +} + +# Dump output to stdout if the tar file is not specified. +$tarFlags .= "O" if (! $tarFile ); + +open (FH, "< $manifestFile") + or die "Could not open $manifestFile: ($!)\n"; + +# Create directory structure with symbolic links to files found in the crawl manifest file. +if (! -d $crawlName){ + mkdir($crawlName); +} + +my $file; +my $dir; +while () { + chomp; + my @parts = split; + if (scalar @parts != 2) { + warn "Illegal format in $manifestFile ($_)\n"; + next; + } + my @chars = split (//,$parts[0]); + next if ($chars[1] eq "-"); + $dir = $flags{"-$chars[0]"}; + mkdir("$crawlName/$dir") if (! -d "$crawlName/$dir"); + $parts[1] =~ /^.+\/([^\/]+)$/; + + if ($1) { + $file = $1; + } else { + warn "Could not recognize a file in: $_\n"; + } + + symlink($parts[1], "$crawlName/$dir/$file") + or die "Could not create link to $parts[1]: ($!)\n"; +} + +# tar up the directory structure. +my $cmd = "tar -$tarFlags $crawlName"; +print "Running $cmd\n" if ($v); +!system($cmd) + or die "Running command $cmd failed: ($!)\n"; + +# clean up + +`rm -rf $crawlName`; + +sub USAGE { + print "heri_manifest_bundle crawl_name manifest_file [-f output_tar_file] [-z] [ -flag directory]\n"; + print "\t -f output tar file. If omitted output to stdout.\n"; + print "\t -z compress tar file with gzip\n"; + print "\t -flag is any upper case letter. Default values C, L, and are R are set to configuration, logs and reports.\n"; + print "Example:\n\theri_manifest_bundle crawl-manifest.txt -f /0/test-crawl/manifest-bundle.tar -z -F filters\n"; +} diff --git a/dist/src/main/bin/xdocToTxt.xsl b/dist/src/main/bin/xdocToTxt.xsl new file mode 100644 index 00000000..882a1267 --- /dev/null +++ b/dist/src/main/bin/xdocToTxt.xsl @@ -0,0 +1,70 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <> + + + + + + diff --git a/dist/src/main/conf/heritrix.cacerts b/dist/src/main/conf/heritrix.cacerts new file mode 100644 index 0000000000000000000000000000000000000000..23c0c87b285917e01cacd742090b2933277d8308 GIT binary patch literal 21653 zcmds92RxN+|K}Wgb8K1J3HPx_*&{_}l4S3_6N)3d2-#U7Ny;cYQL`}_Tk@Adm#8?zg;7#J83@P`ido6geJ)7Hw= z+uhRK)6LS-%+uZ0$B+J#sGBSgaAGc^aK_T401vTz7L>B4hUc{rLBvw=%B8`qUvPM59b8f zk+V=3;}5fz^4mATq9Dfy2bgGxB_se*K{y}`fd33=hy?+(;eY0$hz#_#VPa!I5s?@G zA{>H1V7wHddq7iucIfIYV*_Fu4Pknsq^146r#he5D9_&)X$Xe0G!`n|VAWERo1xYh z6?7iaG_zR!s+F40-n*1`@jxg6>njRzVTVWzCxgsh`49B0@vGWuCkz@$#Wj@I99Q^7 zx*7duM&^q=1DbEa3Lb{dNjlv&J|J!eDWdS*xRMKnKrt~ev#kINfE)+kB;*KY$cjAp z{bNBP7`Ol__?|K?YBcJ+&9Bx76y0Q$9Yow4LDadVZ?BTeqI^MREb% z;QipPH;32FSZ)R#)3;n0S&e6vS?;``&!iHnE3JRMv~=aboa^L+`cK8G$|F6Q_Tzd! zDO^c@%4U?4E$dEDigN+tj7I_^*J(BmB=DA+<>nr)@sBlBP~2AH`N_NOu7T##l;*9B z%RO^3aW3yV*rUC|gS&?dCNOCP1Lo7Nsys-sOk+;GTRRovY0eT}6QG}lMUm+l=frR=#noBajm+x$qaBYST7P?6b{qi3WB;p-}w8BSE4|XYkd0D*FZ5{E>AG`kp?bbwSe#D&mBXYbP1}h5QPq zl2S~YeeRWS77rv}94MezcWGf3m!DPq6u^JlUdXl}^_AhIu>oyUG2c0?1i6@Gx#B~_ zRY4MsQi=1NEm0q8p3BE?GBrVDbcwu|ALN_(L2Ta*6pTQ;OrJ}puyytc_X}K-D z<&OCabrCJQpw2rjHs_oxX2|%= z4)7wEM~1}?><=JCq8SZ-2=E2)B{Tpc|B{gu>Gm*jc6qs=OCDbvBI`D}99hJa{5JSr zR=qlp=H>RBCPID~?3>SI+zM+xoyV2Kz#;=k81VE&_yI{_egF!=ivEG%NTYS!StLNJ zJz|;UCd)HE%M)#}OOrvRIGh(Lf(Q$wN)$iTw959gkW@KQP6KbG%>4W>+4~*6*(Gx5 zVV>divFfW)0amw3dA_Ri)zV)hh;F#u*M2XgQ-EFD(8sZHabsMa9=II6Sk-e>TH!)x zvnGA$iUy*|i!eyiE$yYHRpMPvqb;Ll?O|zt&J1)$Xr>G47iX*J+Gci)7Q_hjiv}RX z>cJ51#t(}DJa;=qFl<0RAOaYvimnoi0%(k!-B4~0&INEFX9!_Ts2SvSb&=O)(Q$V0 zL^?wEZTHCrup-|Gh0*`;MirECWzlwa_CUExaX=h2RRGdmf0OT zc+$+WdCk;!K22dS25Ct6i{jcC4wHO}tBZ(h!fB39Y_71N_qpJ3Naw8>x4VgdNumyJ zhR&*^eq2Xkq+^(Mz{G4xe5)J$>5YAERfdpB3;D^_TL-%(i3=j-^Vu;aCQ@ZEgHICc zhbpbS3eLZ3%Ngu)BtV9Os_EQ)W^4f|v*e!~DClgESah6Wz@IYe)#|t2Z<=%9(2=vF znSQxL?pc)3;KN^TlK>4Dsf7?QymVjTdkda{FspRnH~CXK&3_bl#pI&+#~uwzsHT@B{_r7z;| zRNZ{OdYjP4AB9EYZFgqv?(VSdI1$-FI~c@D1|4TS=r}jNInHb3hkk@2qUskmO$IoO zf*z&lkB(DEN9}L)o05R|7INX}esepA(Xn&`L(qTeK84|8NK3YrwFm;>Py{&y{-+oB zJ6be7GCuCQ@N(WHgtSB7|CLZ8A5E4gP?})DDPU8tcstK?0>fJPWV&V3{%h_NuLki* z#7^XMC?gzdHu64so;-Ou+Tm!u^rd481TExxDJfk0lUq-uoAyFR{L1{qaR)~IHBu^} zasg$Y)-b}1cPB-g&tQlSSyA>mc`<8xtEHcyp>sX#Pr!*3(g#K*V2b!J-&&jc69Ff zIuB#-X+p|^Io-@(dup0_QkLJ-q?dV0jzJi#Wb0UVX&f51vKKYYO zTVgG|(0y9f^88KE=Vod0#8)Qxv=6X19m0j?-&2upxl#`s)3Y9yE|zZY&Q7KdW~T18 z<|s4PHTO^_a>tB8IY1M}3cAyXZAAp*CmZ$*&7T^`!KnZk(xl+Qh)!C%dsw=JuLc%@pwoyY*Xc7OY)&9=PJ$P_c$FWXRBamw1NSsKn!9!>HHByxiXFc5p zBUP^@oXqs@d@7@L&z3O8i)KG=qTLg%tH<6%&WXieCsOYdj%K6@PkbXHe)4GD5}wP{ zAw^%!3#KX2qinCj$d$R432{lP2iyHw2*N7$2>JH~^ev30^UP7!5Ga0OI8EDL%XXD@ z%gVt3@}C8tp9LGfk!1U#!#?3Xr?TQOnRy2(j_EwUbGVKg zOA=HD+L*pWg5B!kpC}A*G>5TI*H9k2KFm`kB^UYGT<`ku{$doRb_+J}Z-R}?*bbW@ zvLKTVfoA&#giKIVs8zFF@1r5&j-{bSGY2}a{)%=3|M#^U(2c_ZvA?R_{NWq;9s0)U zY8|unO!<dMQL$)~`HgctZ13HMwPwhqI8K}kMxTuINC*&7njH=q*3Q!UzJ zZrJdqhcDFkdbwbcFHm(r0G|IL+q$p+r_d98B*@V7n4*=5jm20J!^a}GiTH58LEn4v zb&UM4U(>bm2_J6MOd$)ooG&AlhqW%I9bezu(*I&H1m{UH!+c;EARk4exzECh{js529c|o4ioZ&NBY7NwadGeo>a?3<3-_%-)UVYrrV?Qu- zsyj<$MF|Bo+TFI>3RJZnO>MWL(5|^!&z2p)K|ys8$_Iij;g{yS{l9F!!BiM&=%Fx% zAI-P&_PUT3oEPBP5#m7$zHi5Ze_Q;YGUEThHUExGowhxlMR69VU8huD6KR!AyTud* z8J*`NzT!(U9L(X#?I9_D(3~qH-uKwt1-ekikm>Bf%9jZi^p7c>6bt+&uDnwxb!CmP+41a>nu`0flVZM} z>mN6Gvaj+_newLhu@{c7EGOKsHan0ruTE~#<{W9FZ(6`vtu11R*9v^eJDL!|Y$nfk znTB+!$|9AfK!s;GEy<|E!Nk=tt6<>V=%f>d3Uetj_d7`1Sjtz}Qf{wb@ydWP8z+8D zfO{Jy!;r$QiyVs+sRC3_H*#-B;Cfyn8q^B1A6sdEc{U4yVi?hOa`!#A3k>brVHMT@ znA^Puf8@4vJF%l33h4G58!3v`qY(?-D%x8il>#KT!W_XL6TLqzj>07*B=!_X|6-B{ zZcT{@ioity36w?(-yRAfhyRdOBGS;G9%?V_68Ho|k$Yh;xGxIpGK7C5EfkA^Zx+H$ zTJqkNklD;;OkNdTAh4)bS;nG~VN+3W;G#ot66v}Thw=3?r%+@(xaayRfKEom%J;Ge zcSj|`FneEWx8^y7kHIUk&ySD3=85IBXdg>*ki>{s7QSY=GCNAP`L#k-e;{)xP{JNj zOigE6sarOo0KtT22V(YsLBp%Pa=k!IIuJl$vKNR+2EK;~Kn!If`ZE!)b5IiMKLCS2 z=6yVwT$vmpY&ISqE|LNQKhZ{8O#r#|z;Ev6uuE-4(gO|1$Q1D#lu^h=>9l*Qt@AWy zuoH*xZ4zMEf6)v|{5WgYMvs;v?m4|XMWjPlTZsSy1}wcjlvdDcAVZAk0RXaQ;qmw; z!3{swNMV}1FDfLP9Db27f-hr9rE`~88pw!0$39%B@9K)T!>T^`l2g>O8QwVgL1y%% zXOIv1oU#V((yOHEyima^w@-&g4rlZ*_STyyT-GPqpCDkCeUQ284))w_LGRE^U3a=t zL+n*;X-7UK!p{U(r)P0iOZJFmg;25VinH3Y!5t%r9?QB^jWLq-@(7oSKlOz^uc}37 z(_e{Yh5kY;3yQ1Y#^hhzhy24g@H_Y0Vt6Wjks$(;@;=!xmTdp%$wE6N*(*UrgzZAT zz2vPQAo|Q4pAY0IV&}hyGJE@{U#(2R$Y(p>T@ajQuGHkl^?XUK(bKgMA;WU@Gg<(bZn;K{=e&!=p#D+vvFQO%NOxmu`M@*`}NZvHBk ztyZJpZ9T||p+N~7%>A-XKpu4YBlkY*SOLGXhPC1OckvGxuQ(jc(|YdyI;T!&{&+WM zofqawM640(LVNM%WE-pg3tO5|#eJ!CiW1?51F0CKdXbRi*{oDjE5m~ z>H^Wzl#Cnx%T#gMU0KS3uWi)Y#i(#>3Tsx1?#=To#~a{No1P`u&(W!&l;2b{o-n-H znHMGhoV|`IhU;jlBTj(1#FH!|@vWMRAHH6sIik4d*=Wr8*7QUGMknH!FPxOkeDd%| z=<1SaUUl#2jYRcnUYd}?b9gsk2z&r;A<~e@bdCsm<2-F_+6Evj!Ihl$PU_i!b z1Tf0&4CT8_#bR#$CzL8WfCl+NP#D?v2Ypl6={kz95eEyd@Q_zTi?8kGK#fQqPtt zJkwzB$2&60dPBL>5cA5WL<&wDQD zQRz(FIC@W!us6avh38r_Ew@zrs4RDi!J)3wNfw4nPjhZZPHFXys2x0YtFN-eli`#6 z;#|`r?P~ag?9B)^eis!KduX=U^DSp~akF%^^>o~(QJrVScN&3G0h9-%QS2|}g4TbV zMt_gm`+cu3DQ#B)Ry6#x@Btw=0Y|CM{UsI7DGj1lBn*eH7ke5^d=+`M zlDCNK9Bdsm#9KK+QzV%=Pm+a^GW?N}%Bi|J*dQrjb@QPId= zk6YcFgKA&&#w7Ec#|2Mm_j__{o+4Tyq#G;fElh|M(Co*ppJu)-^k$hC9=;X8>?$us zk?K>lEgkoFQn7=iVgX4NzO5FB3@x>yJ6fb#a2$zCB|3Lt!2m0SgL(l}&IF_;$6y&hT%)-tRE==uEM`HJ;gr%oILmJjWPHnr25Wx6tqfGsiT+rA zopc_Vuvz8pD2eY;dCkT`rXiFRozaPwbnZF5JL#lHPddReBnE6fA%4Id3c;!S0e#3Y zPyZ#)^!n$a(wC4*n$qC*%F#FR&pF`47`d@d{$;V|>6-EFDb8c$L*~IwNgwhy4Vqt_ zD6w)qIaJ3wbNIkbHdRcl5P9GA5B^!y+NIXkuVnkBIQ2`o&mO)@dDgUh?SK_mCh^)$ zcf)Cyg*aAAMdP*gIUbu^S)u3V&dq5Q;GPvnafxc%QEgfBUD4j-(>K2}iWg)Q7s#mS zUq*W`(H`l4DcbvaUvk?)eUJK(o1AD-AKkWl;@evJ&tB>`ao;~}!M`&o9ipOkmwFzb zPVYm=nOzb8po(>(yKQbCLp-(09PJSiN$n+GJDe+00@HeE3zuS>geVRKw#$57wx(*B2F?`pAzu;bjQ$jjrZPhPan0>*zO! zW7k|HuW*;i$BkT)SD%zk+b8y?oD}DggVc}p*WOWk5as6^>&h(lw0N5s899lW?>{Sg zp@im%aOE;y@OlfT_@?^9MQVA4$!Y`TFOM3@nHP>0aVxjfKaoHGVD0uPr>=6XF!Iry zU^@FiwHzXfJ5xD#WftmHEmJNxG)r@o$0b+C3cBLXhZxOja^(`|vaMD>%f+RXM-l5= zu@YFGBaU)Wi`fBMSv!QnG6BVGLy%O8+b&Anywd6iBk}K)!-5*NG4hgsuFCh9LiR7J zeD|(RXW>R(nP;oH0xl?sDh1i{gkXs}I20EIz~TSuvH~&pqdy>|W*vx0iUGPf0!Sbx zCdM80W49X(TA(rGfh(O!1>e-cmS`_4 z-?))N?`}(-Gl#lUOImrLs{%QiC7nlJVhf&#_*!D2MlzO!>)G!=T#0wWBpBdX^*xef z8!{>)!xVDH$A_wB4kL+!I(M}%l@O5}4UN(I7Iy*jfr3;#){lzrOYX5=iS)`_md>p4# z3Rr_8Xj)Rkb(pFGg6SI%9hWXnE%R%PJws~&8GVI=M{eq8KFXU0hq`!mWOHXORi~J{ z&pM7D=%1y$#oH)K-o`tFd+A1Bx*=Vf9>uaQPc!yx3B`fPB`Cw@WGq=|kz{Fo8nzfk zcGD77Wmk>_^)($ywwQ&yz%08kn@8s$`yb@zcHbQ?qBO1uZ5#CS_uVYq*E-RUd&j6h zIIWGBSg0SXeL__h#_RP(dMY)Xl)+{I12wvnOU+5?3Zp&_h? zu%4F3+4#2}no8;s-+W)!t2xhqO#iO3a~prXNuS#Zw;DC2bknNZAVe&*y~Eb8VavUP zodzDZj?QjqMZ73$G+TA5VS2|JL483>bQW~)#@p5i?B%LN2RkxgHb8Ums!HF=iwI!& z^53oj2Lmo-5KRapP}r&6h3^MgkS{`EbUSZk`98fixHq4xfC->6TJ@u9Iiv?YsDZ;-b;hQM^1U7Q@%i%ogEeJN9$)zckA- z+qfUrM=>Kl!=DM$YpP~KYwZ#$v1w5Fjuo9PqUtT zdigd-XhcfFzFXE6GDKaUSG8dRj~TjLT;{LTQ9Q+Ymh!gusgTdv?v?kI3pR@KM_?Uw zk8|!CnQWBhVz1vH#61P+JVl#izI?;{O*GN@GbK1aAA_f7I-z%sf~1W%sb*~-tV)Qw zdA6R_R{z)%eX#QIcnh133^RKYt08VOPferb`$?w|N&7?(bD6v>~nl>$c( z87-%!mo}&S@i*tBAIGCS=J?gc(VpJC#$tlG{N?;&27fg}`J!f+d+g(pQR>JPLCEbW z+QBI@@(&k8ZbuhN193iWLWgf!k}j`DrJJ>?C?3Q-Q_WZ^*rC!P&fm6y7!b5$D*bS% zYkd;INLSA;KW#2?YdfdiYU0_|78LMxZHJ;*3t+%73;yVR3mBj=AJ8d^l3ZKRIQq*S z@FRdSilvk?KW*iq+INh>Mj=%vD`$bdHpJ1Jg?{K2`m<5Qzw8wPJKMkwCAfs3un?+x ze|so`93l&AcQuCYz1#o4vDV-1hBr$l8^LLY3%H!4eaGlFm9m8&_dm+hn75e@`kLW+ zuJuubR$O7zNvMiN`kg=qgFb&TZKc1 zv4Uc!ORr7uQ?AZAeZ@P4;z;u9lE@H|C@#dFOfsAjkqXcCE0HU)o5aPuerz>^Ln`5E za=r9*t$iQs`4vAGSu5wqj!X6JqoWiDx<<0#9Q+IKmBeRPaP*<>W<_N1E3&l~Zw#dy zG?@D+6gJf+a5(d>(d8>1@+vWWk1~uqC7x*7M7#cb$MGG@2oVPZ7hw>4*CAw(J60Et zD{UB^bUu_sXEeA=cG+WFoCRP)T0#OCjkb;)i=wHA=~g#6*l4@mXv7a3MA=Fh#~*Id zkPwE$f$#0P;J(qXOFaMn8$et`cM7P&nsBhNCKz%7Ble}wqq`TS4f4Y;o-E+N^dh=* zH|n6+%NY@G3ti}VkgQfq?G?O{h(Np!>JYILQ^l;1J6<-36zr^HEyp;l_Uk!>;a>TS zIOQ0GYv}@W^^(4JR&7)>TrcTJCF4yVdKmvv@>8iyR9&jN2i+xY1B(+SQXO5akBkrU zbvHBlqM``&TSJz2Ucj&%yzsWIoA<8DDRkr&ubMLNLuEN3()g)wDeko!d=k4+ zZVD|sO>HaHsE9`trbA?(sM;hgGT=~Ioj+_?Eh%u-r0mu)^+lG?FO2;ye4_?1413nj zK9!(8zZ2tv$k~pvAOM2w2iF# z4#1%LV05etg6zxDvu_0U&>436b@8#-!$atm0*DwfV$Eo`Xs2^aum?GP9XQ|!FH z-La-1hs>M7UPKY}LkZA_f6F!gRj}LloYoxpxSC=F-=J= z^0Xq9X?K#8WBE0nE`%yPqK-ewr#TnS7vOX9&f*7S`wooAO3D>VDv{ngx>&}T;|)*WD{Mtns^qKBSD8z z9uJX-Atc!l=Wl1kPWhxR?M#ZxSa^`>iiyO?ny3y;qSJ_zxBqY+WsTqrEtFdu%XR#= z(BZe@)wrTTW~t|YpU_cIOZe3EP;!@u0dwjd%upMT!6 z;_36_yShd9e8V20ugEBm-MGLR-6$T-xQt|a%CRfn2h8HKxUAFLTIr2crXH2#v<}MA zV;{eE?88y#6WaIKuZt1PSa==aKlt&hlxKA9n(OPP!(tt!*zsnSxkKTHMSaE39!V_7 zWx2wi+d}lXBv}cg=_R?={{3%mOqWp|!MB(;((w3q!pDiEa&eYEv-l$V=TKp5qINJ)5Sae%Uq!(SPosAgPCV$L$%9!sUUYHa_u~( z`kqWaP#0#pWvDJ>9E&dyW_9LlViThm>{i$K0yRFlNLN8WyQ|B=Bt61jn)~geTD(S{ z@4DIsG=yg#V_OAWe>NesO+fS(Q<3fTRLjZxI{~Xe0+xXUOy43P1hec%{<-Yt-><7u z{&0&a6R>ZKs#JTFuds;hbe9Ulg;CviV0Wny*la5NEoJ@xzoUP}?FST2G7)}0iJ0%1 zswE5{OWulEM;M}fRkeL8d{yoh#$9u+PFUl$vox&O7w7)4Hk@F8z=xCTm$9o|G7=@Q z`m#d}^TwYR)(NF_i5VU%7z=32Wi$Em)r~(ct?A`sEWtT%lQXk-LP?b@nGU58q&z&s znEZZnvAxlUruWV@-30$9ekQfS`BWl|vVG(a-s`e*^RZpen{VZNF}Qw#)ab1xuHG}; z`yG1kjNqURdJ@vX7uZ2qO!!QyU<{AwZr;vto{kOleWopCnimSY%!Vs;uH_8AB^iRGVB~V|JCae#>s< zBs*zO_f-tB?t2@BKfYiiyg8Jxgu8t6(8q$K#n;wi*RIV@2(soM^=cT<5Fa1z3K{Ea z_Bx-`Q~ctJdT9)s!=;FsX}>wjxF?PC%t!eIUCnvTV>luBWdV${BhM@R`$Nr}-?lk$ z7K2}_6&b^ecwJf{)AulqKG^5vLg}{Fs9d+Kd-d2UEnMegpI^|9ym#BhF+Ds(eTLri zaZ-L4gt~_)qLyQ9hVMbrl)gBfU!~Ya|41^PJ5E#LMZ5E4m;!>8G}uQiCj?$TG}6fp zoJ!Fl2=d^&__R5W+RHf3lzTq-rbKnmrQW;}7^5z}F#CBH#CfX=Sv9jKz|Hj}6LS@aJd~Ufz6@8gzz*#3wUf^wJu4nK%Ev z_wQ6bV$Dah-JdQT*Qvn%ta64oWZ`B`qm)Ql2eGzX&r+=C+bjZ2z0ik=guX(-WR={s zAB3ilkoTO9QSsW}`Jwg9WZv<8X$didsWl$>d2XipMXZtwhYv#a@H-w2)R#8tirt>z z%qmfKb$?18_Tj;##pgD1dUsiaX4QkF<#W;gHUU`Tgbbuooy(c`9!I?uz8{e6nC{gX z%OhJH-yxP`P2vB>top7AIJ3w4gvuN2VGRC|Y^CjyAAfBB<)*_4IU8rz z1WF&w%o9%Oi8Ho45?_B{b7{UI;0C+!*Tn={@{sGI%VLwdvRxL;^~Nsp_Iwge>Dt&Y zC+*xiBHd(EG%wz6isWc18ojAQ9iMBbGfkUz#rxLTv-GR?6ID)yadDSEeaqh?O_D`i zv0o87+HPJXCwYB2hwrl&6P9kWx1ekfub+BVZfemvC)|gZ-+$;4w>|xcNY5r?W2XIW zbJ*(Q3;&0~_Z6)%G^Lj~p%&9;&%ZO0-Y2Hg+&!K1QS?yaV+l9xI698{9x*x6N6euS zU-KVM6yN$O`d0Ow6mtyBp#DlPhrVin3TY$sZoz16d0qYi6tS6im11n2Y-9Q3 zd0%#433nI{i38ak4JK==TMWniscaBk>iw7JeQnohK!2*y*p90I$!q^^d$nqk zCK_T{EPUdS>SL|0;hXJZSK22L?sD2D*V3-jaC+ZqDt^ZqCs;Zp6e%gtznJ1<;GUFi zIQ1pMoxyOjzBa(&#jT}o-n!4L$@%74#VQce&M_@XibD<&2D(8m{f`==TiukN<(;Z~ zA(SSQoiGzD#JW$KN!rB8E7dx0b}?GCkZh?G?*g*j|6h+q`$u-o^oq zaC=Up1;X1(12BTAmESaN+*jFVNeXmJ_YqU7I>lV&qJ+;q`OK%Z^1i7yOOBZBe(Wk8 zhII{Q;fpt}>jteBPu?UXwPnkWU`EVw;VxKWMkw;(SbA#Lg^;nZ8avpNI^4zISGbr( zEBwSW$9>;Ib zpK0t8j-d(HQzh+})N?X87L#!{ekMXx zR%!Jwk4F>)ari0P4wAluIOva+Y+^30rRY#Y21GwpvVDiz?z**aha(dH{%}OWt^IrO zG(_-FIk>Q}2(pq5c~;`L;op9}-vMb%LgFfuHVy-~=t@a8Ps40f33<)N%;bHK$5%G7 zCwtRyYAKp!NEW->*-OZW+THOZI*Sh`;S)1yDwkZ$Rf;sTsSajfaS_ayeUY1B{KQpE z2J2(F9J>TAJrlpFekKLs_>JL)+tXsM&-XQpNmeFm-z;dbV~;zY>f2gp*FDTyvt=3o zcqQ9k{qld5nSZEc+w~lNdBTQfI^lGoQ6s&eAE)9-x zr7h>vGCLaaRC}$i<-S@X9BAM?`8EcaegS`ERHEHx8$7Gvd~9l2-@0s~xbgo0la@@4 literal 0 HcmV?d00001 diff --git a/dist/src/main/conf/jndi.properties b/dist/src/main/conf/jndi.properties new file mode 100644 index 00000000..f413dbf2 --- /dev/null +++ b/dist/src/main/conf/jndi.properties @@ -0,0 +1,22 @@ +# This properties file gets built into the heritrix jar. To override these settings, +# put a jndi.properties file in $HERITRIX_HOME (or in $JAVA_HOME/lib) with your +# alternatives configuration. + +# We do not have a default. If no JNDI provider, we'll just log not context found. + +# To use JBOSS JNP as a remote JNDI Service Provider, add to the lib heritrix +# lib directory the following jars from JBOSS: +# +# jboss-common-client.jar +# jnp-client.jar +# +# Then use configuration like the below with java.naming.provider.url pointing +# to your JBOSS JNP instance. Note, if you are having trouble connecting to a +# remote JNP instance -- and you're getting exceptions mentioning 127.0.0.1 +# though you've setup a jnp url with the remote machines' IP -- then your jboss +# server /etc/hosts file is not set correctly. See +# http://www.theserverside.com/discussions/thread.tss?thread_id=30166 +# +# java.naming.factory.initial=org.jnp.interfaces.NamingContextFactory +# java.naming.provider.url=jnp://localhost:1099 +# java.naming.factory.url.pkgs=org.jboss.naming:org.jnp.interfaces diff --git a/dist/src/main/conf/jobs/profile-defaults/defaults.xml b/dist/src/main/conf/jobs/profile-defaults/defaults.xml new file mode 100644 index 00000000..060761e5 --- /dev/null +++ b/dist/src/main/conf/jobs/profile-defaults/defaults.xml @@ -0,0 +1,165 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml b/dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml new file mode 100644 index 00000000..2fe2c8db --- /dev/null +++ b/dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml @@ -0,0 +1,50 @@ + + + + + + + + + + + + +metadata.operatorContactUrl=ENTER_AN_URL_WITH_CONTACT_INFO_HERE_FOR_WEBMASTERS_AFFECTED_BY_YOUR_CRAWL +metadata.description=basic crawl +metadata.jobName=basic + + + + + + + + + + +# URLS HERE +http://example.example/example + + + + + + + + + diff --git a/dist/src/main/conf/logging.properties b/dist/src/main/conf/logging.properties new file mode 100644 index 00000000..cfaaeeea --- /dev/null +++ b/dist/src/main/conf/logging.properties @@ -0,0 +1,117 @@ + +# Basic logging setup; to console, all levels. +# Note, after startup Heritrix adds a special logging handler called SinkHandler +# that keeps around all instances of severe and warning messages for display by +# the Heritrix alerting system. We used to initialize it here with the +# ConsoleHandler but it wasn't being found when Heritrix was packaged as a webapp +# deployed in Tomcat. +handlers = java.util.logging.ConsoleHandler +java.util.logging.ConsoleHandler.level = ALL +java.util.logging.ConsoleHandler.formatter= org.archive.util.OneLineSimpleLogger + +# Default global logging level: only warnings or higher +.level= WARNING + +# Enable frontier INFO logging +# org.archive.crawler.frontier.BdbFrontier.level = FINER +# org.archive.crawler.frontier.BdbMultipleWorkQueues.level = INFO +# org.archive.crawler.frontier.AbstractFrontier.level = INFO +# org.archive.crawler.frontier.WorkQueueFrontier.level = INFO + +# Currently necessary (?) for standard logs to work +crawl.level= INFO +runtime-errors.level= INFO +local-errors.level= INFO +uri-errors.level= INFO +progress-statistics.level= INFO +recover.level= INFO + +# Set the selftest console logger level. +org.archive.crawler.admin.SelftestCrawlJobHandler.level = INFO + +# Enable info level on CrawlJob so can see it registered jobs w/ mbeanserver +org.archive.crawler.admin.CrawlJob.level = INFO + +# Enable the following lines to watch authentications running. +# org.archive.crawler.fetcher.FetchHTTP.level = FINE +# org.archive.crawler.prefetch.PreconditionEnforcer.level = FINE + +# HttpClient is too chatty... only want to hear about severe problems +# For more on httpclient logging, +# see http://jakarta.apache.org/commons/httpclient/logging.html +org.apache.commons.httpclient.level = SEVERE + +# If you need verbose console logging of HttpClient traffic, uncomment the +# following line: +# httpclient.wire.level = FINE + +# Enable ARCWriter.level line below to see logging of the opening and closing +# of arc files in console output (Console goes into heritrix_out.log). If you'd +# rather have console output, including the opening and closing of ARCs, go to +# a file, enable the FileHandler logger -- see the commented out 'handlers' +# line above -- and enable the lines below related to FileHandler (Configure to +# your preference -- the '%h' in the below means value of java.home system +# property, usually your home directory). There does not seem to be a way +# other than in code to configure only ARCWriter writing to the FileHandler. +# Even if you do do code changes, its awkward making the log show in each +# individual jobs log directory. Note, the closing of arc files on shutdown is +# not logged. +# org.archive.io.arc.ARCWriter.level = INFO + +# Enable to watch authentication progress amongst other details on http fetch. +# org.archive.crawler.fetcher.FetchHTTP.level = FINE + +# Enable logging of already seen memory grows; this growing is probably +# main culprit for OOMEs. Log goes into heritrix_out.log. +org.archive.util.MemLongFPSet.level = INFO +# org.archive.crawler.url.Canonicalizer.level = INFO + +# Enable logging of uris rejected by scope by setting level to INFO. +# org.archive.crawler.postprocessor.LinksScoper.level = INFO + +# Below values are used by classes that set their own FileHandler as default +# values. The pattern value is used as a suffix. Note, the below configuration +# limits log file size (to 500000 bytes) and because count is 1, old log data +# will be lost. +java.util.logging.FileHandler.level = ALL +java.util.logging.FileHandler.pattern = %u.log +java.util.logging.FileHandler.formatter = org.archive.util.OneLineSimpleLogger +java.util.logging.FileHandler.limit = 500000 +java.util.logging.FileHandler.count = 1 + +# org.archive.crawler.admin.StatisticsTracker.level = INFO + +# Enable info logging so can see note on checkpoint completion in +# heritrix_out.log. +org.archive.crawler.framework.CrawlController.level = INFO + +# AdaptiveRevist module +# org.archive.crawler.frontier.AdaptiveRevisitFrontier.level = FINE +# org.archive.crawler.frontier.AdaptiveRevisitHostQueue.level = FINE +# org.archive.crawler.frontier.AdaptiveRevisitQueueList.level = FINE +# org.archive.crawler.processor.ChangeEvaluator.level = FINE +# org.archive.crawler.processor.WaitEvaluator.level = FINE +# org.archive.crawler.processor.HTTPContentDigest.level = FINE + +# ServerCache creation of crawl host (Make it FINER if you want to see +# crawl server creation also). +# org.archive.crawler.datamodel.ServerCache.level = FINE + +# Enable to see average alreadyseen lookup times. +# org.archive.crawler.util.BdbUriUniqFilter.level = INFO + +# Enable to see duplicated-detection stats on each merge, +# if using an FPMergeUriUniqFilter +# org.archive.crawler.util.FPMergeUriUniqFilter.level = INFO + +# Enable to see report of duplicates encountered every 50K unique adds +# org.archive.crawler.util.SetBasedUriUniqFilter.level = FINE +# Used to set a log of all URIs presented to filter, for +# later comparison profiling +# org.archive.crawler.util.SetBasedUriUniqFilter.profileLogFile = uriUniq.log + +# Leave the below enabled so can see launch info messages on the console. +org.archive.crawler.Heritrix.level = INFO + +# Leave RecoveryJournal level at INFO to see recovery progress +org.archive.crawler.frontier.RecoveryJournal.level = INFO diff --git a/dist/src/main/extras/pagerank/GenGraph.java b/dist/src/main/extras/pagerank/GenGraph.java new file mode 100644 index 00000000..e95a2a11 --- /dev/null +++ b/dist/src/main/extras/pagerank/GenGraph.java @@ -0,0 +1,149 @@ +package org.apache.hadoop.examples; + +import java.io.*; +import java.util.*; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.mapred.JobClient; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.Mapper; +import org.apache.hadoop.mapred.OutputCollector; +import org.apache.hadoop.mapred.Reducer; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.MapReduceBase; + +public class GenGraph { + +public static class MapClass extends MapReduceBase implements Mapper { + + private Text index = new Text(); + private Text toUrl = new Text(); + private Text fromUrl = new Text(); + + + public void map(WritableComparable key, Writable value, OutputCollector output, Reporter reporter) + + throws IOException { + + String indexedUrlToFrom = ((Text)value).toString(); + String[] splitString = indexedUrlToFrom.split("\t"); + + + index.set("-" + splitString[0]); + + toUrl.set(splitString[1]); + + output.collect(toUrl, index); + + //System.out.println("map output (tourl,index) = (" + toUrl + "," + index + ")"); + + + index.set(splitString[0]); + + for(int i = 2; i < splitString.length; i++) { + + String fromUrlString = splitString[i]; + + //to avoid self loops + if(!(fromUrlString.equals(splitString[1]))) { + + fromUrl.set(fromUrlString); + + output.collect(fromUrl, index); + } + + } + + } + +} + +public static class Reduce extends MapReduceBase implements Reducer { + + public void reduce(WritableComparable key, Iterator values, OutputCollector output, Reporter + + reporter) throws IOException { + + TreeSet treeSet = new TreeSet(); + + Text marker = null; + + boolean foundMarker = false; + + String currentPIN = null; + + while (values.hasNext()) { + + currentPIN = values.next().toString(); + + if(currentPIN.charAt(0) == '-') { + + if(foundMarker == false) { + marker = new Text(currentPIN.substring(1)); + foundMarker = true; + } + + } else { + + treeSet.add(new Integer(currentPIN)); + + } + + } + + if(foundMarker) { + + Iterator iterator = treeSet.iterator(); + + String finalOutput = ""; + + //read in first value, again tab hack + if(iterator.hasNext()) { + + finalOutput += iterator.next().toString(); + } + + + while(iterator.hasNext()) { + + finalOutput += '\t' + iterator.next().toString(); + + } + + output.collect(marker, new Text(finalOutput)); + //System.out.println("reduce output (marker,finaloutput) = (" + marker + "," + finalOutput + ")"); + + } + + } + +} + +public static void main(String[] args) throws IOException { + + JobConf conf = new JobConf(GenGraph.class); + + conf.setJobName("genGraph"); + + conf.setOutputKeyClass(Text.class); + + conf.setOutputValueClass(Text.class); + + conf.setMapperClass(MapClass.class); + + //conf.setCombinerClass(Reduce.class); + + conf.setReducerClass(Reduce.class); + + conf.setInputPath(new Path(args[0].trim())); + + conf.setOutputPath(new Path(args[1].trim())); + + JobClient.runJob(conf); + +} + +} diff --git a/dist/src/main/extras/pagerank/PageRank.java b/dist/src/main/extras/pagerank/PageRank.java new file mode 100644 index 00000000..fa15f43b --- /dev/null +++ b/dist/src/main/extras/pagerank/PageRank.java @@ -0,0 +1,170 @@ +package org.apache.hadoop.examples; + +import java.io.*; +import java.util.*; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.mapred.JobClient; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.Mapper; +import org.apache.hadoop.mapred.OutputCollector; +import org.apache.hadoop.mapred.Reducer; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.MapReduceBase; + +public class PageRank { + +public static class MapClass extends MapReduceBase implements Mapper { + + private Text toUrl = new Text(); + private Text toUrlList = new Text(); + private Text fromUrl = new Text(); + + private Text prValue = new Text(); + + + public void map(WritableComparable key, Writable value, OutputCollector output, Reporter reporter) + + throws IOException { + + String inputString = ((Text)value).toString(); + + String[] splitString = inputString.split("\t"); + + fromUrl.set(splitString[0].trim()); + + String prValString = splitString[1].trim(); + + String toUrlListString = splitString[2].trim(); + + toUrlList.set(toUrlListString); + + String newToUrlListString = toUrlListString.substring(2); + + if(!(newToUrlListString.equals(""))) { + + String[] toUrls = newToUrlListString.split(","); + + + double outdegree = toUrls.length; + + + Double prDoubleValue = new Double(prValString); + + if(outdegree > 0 ) { + + double val = prDoubleValue.doubleValue() / outdegree; + + String valString = Double.toString(val); + + prValue.set(valString); + + + for(int i = 0; i < outdegree; i++) { + + toUrl.set(toUrls[i].trim()); + + output.collect(toUrl,prValue); + + // System.out.println("tourl - " + toUrls[i] + " - " + valString); + } + + } + + } + + output.collect(fromUrl, toUrlList); + + //System.out.println("fromurl - " + splitString[0] + " - " + toUrlListString); + + + } + +} + +public static class Reduce extends MapReduceBase implements Reducer { + + public void reduce(WritableComparable key, Iterator values, OutputCollector output, Reporter + + reporter) throws IOException { + + Text toUrlList = null; + + + String toUrlListString = "O:"; + + double value = 0.0; + + + while (values.hasNext()) { + + String readValue = values.next().toString(); + + if(readValue.startsWith("O:")) { + + toUrlListString = readValue; + + //System.out.println("tourllist" + toUrlListString); + } else { + + Double val = new Double(readValue); + + value+=val.doubleValue(); + + //System.out.println("value is" + val); + + } + + } + + value*=0.85; + value+=0.15; + + String finalOutput = ""; + + //no outlink, delete accumulated values, keep random jump + if(toUrlListString.equals("O:")) { + + value=0.15; + + } + + finalOutput+=Double.toString(value)+"\t"+toUrlListString; + + //System.out.println("final op" + finalOutput); + + output.collect(key, new Text(finalOutput)); + + } +} + + + +public static void main(String[] args) throws IOException { + + JobConf conf = new JobConf(PageRank.class); + + conf.setJobName("pageRank"); + + conf.setOutputKeyClass(Text.class); + + conf.setOutputValueClass(Text.class); + + conf.setMapperClass(MapClass.class); + + //conf.setCombinerClass(Reduce.class); + + conf.setReducerClass(Reduce.class); + + conf.setInputPath(new Path(args[0].trim())); + + conf.setOutputPath(new Path(args[1].trim())); + + JobClient.runJob(conf); + +} + +} diff --git a/dist/src/main/extras/pagerank/README.txt b/dist/src/main/extras/pagerank/README.txt new file mode 100644 index 00000000..c1c7e524 --- /dev/null +++ b/dist/src/main/extras/pagerank/README.txt @@ -0,0 +1,9 @@ +This directory contains utility classes and scripts which can be used, together +with other tools like Hadoop, to perform an offline PageRank-calculation of a +crawl's revealed link graph. + +For more information, visit: +http://webteam.archive.org/confluence/display/Heritrix/Offline+PageRank+Analysis+Notes. + +Note that the Java files in this directory are modifications of Hadoop +example code, and are thus published under the Apache license. \ No newline at end of file diff --git a/dist/src/main/extras/pagerank/assignUrlIndex.pl b/dist/src/main/extras/pagerank/assignUrlIndex.pl new file mode 100755 index 00000000..28f3fb98 --- /dev/null +++ b/dist/src/main/extras/pagerank/assignUrlIndex.pl @@ -0,0 +1,17 @@ +#!/usr/bin/env perl + +$prev = ""; +$index = -1; +while() { + $line = $_; + chomp($line); + ($from,$to) = split(/ /,$line); + + if($from ne $prev) { + ++$index; + } + + print "$index\t$from\t$to\n"; + $prev = $from; +} + diff --git a/dist/src/main/extras/pagerank/pageRankSetup.pl b/dist/src/main/extras/pagerank/pageRankSetup.pl new file mode 100755 index 00000000..f5d11d9f --- /dev/null +++ b/dist/src/main/extras/pagerank/pageRankSetup.pl @@ -0,0 +1,14 @@ +#!/usr/bin/env perl + +while() { + $line = $_; + chomp($line); + + ($fromUrl,@toUrls) = split(/\t/,$line); + + $toUrlString = "O:"; + $toUrlString.= join ",",@toUrls; + + print "$fromUrl\t1\t$toUrlString\n"; +} + diff --git a/dist/src/main/extras/pagerank/run-pr.sh b/dist/src/main/extras/pagerank/run-pr.sh new file mode 100755 index 00000000..4c014ca8 --- /dev/null +++ b/dist/src/main/extras/pagerank/run-pr.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env sh + +#assuming normalized input + +lines=`cat input/* | wc -l`; + +for f in `seq 1 20`; do +sh bin/hadoop jar hadoop-0.14.3-examples.jar pageRank input output + +rm input/* + +total=`cat output/* | cut -f2 | ~vinay/tools/sum_col`; +cat output/* | ~vinay/tools/normalizePageRank.pl $total $lines > input/data.pr.$f + +#mv output/* input/ + +rm -rf output/ + +done diff --git a/dist/src/main/licenses/ant.LICENSE b/dist/src/main/licenses/ant.LICENSE new file mode 100644 index 00000000..8b5861db --- /dev/null +++ b/dist/src/main/licenses/ant.LICENSE @@ -0,0 +1,203 @@ +/* + * Apache License + * Version 2.0, January 2004 + * http://www.apache.org/licenses/ + * + * TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + * + * 1. Definitions. + * + * "License" shall mean the terms and conditions for use, reproduction, + * and distribution as defined by Sections 1 through 9 of this document. + * + * "Licensor" shall mean the copyright owner or entity authorized by + * the copyright owner that is granting the License. + * + * "Legal Entity" shall mean the union of the acting entity and all + * other entities that control, are controlled by, or are under common + * control with that entity. For the purposes of this definition, + * "control" means (i) the power, direct or indirect, to cause the + * direction or management of such entity, whether by contract or + * otherwise, or (ii) ownership of fifty percent (50%) or more of the + * outstanding shares, or (iii) beneficial ownership of such entity. + * + * "You" (or "Your") shall mean an individual or Legal Entity + * exercising permissions granted by this License. + * + * "Source" form shall mean the preferred form for making modifications, + * including but not limited to software source code, documentation + * source, and configuration files. + * + * "Object" form shall mean any form resulting from mechanical + * transformation or translation of a Source form, including but + * not limited to compiled object code, generated documentation, + * and conversions to other media types. + * + * "Work" shall mean the work of authorship, whether in Source or + * Object form, made available under the License, as indicated by a + * copyright notice that is included in or attached to the work + * (an example is provided in the Appendix below). + * + * "Derivative Works" shall mean any work, whether in Source or Object + * form, that is based on (or derived from) the Work and for which the + * editorial revisions, annotations, elaborations, or other modifications + * represent, as a whole, an original work of authorship. For the purposes + * of this License, Derivative Works shall not include works that remain + * separable from, or merely link (or bind by name) to the interfaces of, + * the Work and Derivative Works thereof. + * + * "Contribution" shall mean any work of authorship, including + * the original version of the Work and any modifications or additions + * to that Work or Derivative Works thereof, that is intentionally + * submitted to Licensor for inclusion in the Work by the copyright owner + * or by an individual or Legal Entity authorized to submit on behalf of + * the copyright owner. For the purposes of this definition, "submitted" + * means any form of electronic, verbal, or written communication sent + * to the Licensor or its representatives, including but not limited to + * communication on electronic mailing lists, source code control systems, + * and issue tracking systems that are managed by, or on behalf of, the + * Licensor for the purpose of discussing and improving the Work, but + * excluding communication that is conspicuously marked or otherwise + * designated in writing by the copyright owner as "Not a Contribution." + * + * "Contributor" shall mean Licensor and any individual or Legal Entity + * on behalf of whom a Contribution has been received by Licensor and + * subsequently incorporated within the Work. + * + * 2. Grant of Copyright License. Subject to the terms and conditions of + * this License, each Contributor hereby grants to You a perpetual, + * worldwide, non-exclusive, no-charge, royalty-free, irrevocable + * copyright license to reproduce, prepare Derivative Works of, + * publicly display, publicly perform, sublicense, and distribute the + * Work and such Derivative Works in Source or Object form. + * + * 3. Grant of Patent License. Subject to the terms and conditions of + * this License, each Contributor hereby grants to You a perpetual, + * worldwide, non-exclusive, no-charge, royalty-free, irrevocable + * (except as stated in this section) patent license to make, have made, + * use, offer to sell, sell, import, and otherwise transfer the Work, + * where such license applies only to those patent claims licensable + * by such Contributor that are necessarily infringed by their + * Contribution(s) alone or by combination of their Contribution(s) + * with the Work to which such Contribution(s) was submitted. If You + * institute patent litigation against any entity (including a + * cross-claim or counterclaim in a lawsuit) alleging that the Work + * or a Contribution incorporated within the Work constitutes direct + * or contributory patent infringement, then any patent licenses + * granted to You under this License for that Work shall terminate + * as of the date such litigation is filed. + * + * 4. Redistribution. You may reproduce and distribute copies of the + * Work or Derivative Works thereof in any medium, with or without + * modifications, and in Source or Object form, provided that You + * meet the following conditions: + * + * (a) You must give any other recipients of the Work or + * Derivative Works a copy of this License; and + * + * (b) You must cause any modified files to carry prominent notices + * stating that You changed the files; and + * + * (c) You must retain, in the Source form of any Derivative Works + * that You distribute, all copyright, patent, trademark, and + * attribution notices from the Source form of the Work, + * excluding those notices that do not pertain to any part of + * the Derivative Works; and + * + * (d) If the Work includes a "NOTICE" text file as part of its + * distribution, then any Derivative Works that You distribute must + * include a readable copy of the attribution notices contained + * within such NOTICE file, excluding those notices that do not + * pertain to any part of the Derivative Works, in at least one + * of the following places: within a NOTICE text file distributed + * as part of the Derivative Works; within the Source form or + * documentation, if provided along with the Derivative Works; or, + * within a display generated by the Derivative Works, if and + * wherever such third-party notices normally appear. The contents + * of the NOTICE file are for informational purposes only and + * do not modify the License. You may add Your own attribution + * notices within Derivative Works that You distribute, alongside + * or as an addendum to the NOTICE text from the Work, provided + * that such additional attribution notices cannot be construed + * as modifying the License. + * + * You may add Your own copyright statement to Your modifications and + * may provide additional or different license terms and conditions + * for use, reproduction, or distribution of Your modifications, or + * for any such Derivative Works as a whole, provided Your use, + * reproduction, and distribution of the Work otherwise complies with + * the conditions stated in this License. + * + * 5. Submission of Contributions. Unless You explicitly state otherwise, + * any Contribution intentionally submitted for inclusion in the Work + * by You to the Licensor shall be under the terms and conditions of + * this License, without any additional terms or conditions. + * Notwithstanding the above, nothing herein shall supersede or modify + * the terms of any separate license agreement you may have executed + * with Licensor regarding such Contributions. + * + * 6. Trademarks. This License does not grant permission to use the trade + * names, trademarks, service marks, or product names of the Licensor, + * except as required for reasonable and customary use in describing the + * origin of the Work and reproducing the content of the NOTICE file. + * + * 7. Disclaimer of Warranty. Unless required by applicable law or + * agreed to in writing, Licensor provides the Work (and each + * Contributor provides its Contributions) on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied, including, without limitation, any warranties or conditions + * of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + * PARTICULAR PURPOSE. You are solely responsible for determining the + * appropriateness of using or redistributing the Work and assume any + * risks associated with Your exercise of permissions under this License. + * + * 8. Limitation of Liability. In no event and under no legal theory, + * whether in tort (including negligence), contract, or otherwise, + * unless required by applicable law (such as deliberate and grossly + * negligent acts) or agreed to in writing, shall any Contributor be + * liable to You for damages, including any direct, indirect, special, + * incidental, or consequential damages of any character arising as a + * result of this License or out of the use or inability to use the + * Work (including but not limited to damages for loss of goodwill, + * work stoppage, computer failure or malfunction, or any and all + * other commercial damages or losses), even if such Contributor + * has been advised of the possibility of such damages. + * + * 9. Accepting Warranty or Additional Liability. While redistributing + * the Work or Derivative Works thereof, You may choose to offer, + * and charge a fee for, acceptance of support, warranty, indemnity, + * or other liability obligations and/or rights consistent with this + * License. However, in accepting such obligations, You may act only + * on Your own behalf and on Your sole responsibility, not on behalf + * of any other Contributor, and only if You agree to indemnify, + * defend, and hold each Contributor harmless for any liability + * incurred by, or claims asserted against, such Contributor by reason + * of your accepting any such warranty or additional liability. + * + * END OF TERMS AND CONDITIONS + * + * APPENDIX: How to apply the Apache License to your work. + * + * To apply the Apache License to your work, attach the following + * boilerplate notice, with the fields enclosed by brackets "[]" + * replaced with your own identifying information. (Don't include + * the brackets!) The text should be enclosed in the appropriate + * comment syntax for the file format. We also recommend that a + * file or class name and description of purpose be included on the + * same "printed page" as the copyright notice for easier + * identification within third-party archives. + * + * Copyright [yyyy] [name of copyright owner] + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/dist/src/main/licenses/ant.NOTICE b/dist/src/main/licenses/ant.NOTICE new file mode 100644 index 00000000..5c857d41 --- /dev/null +++ b/dist/src/main/licenses/ant.NOTICE @@ -0,0 +1,20 @@ + ========================================================================= + == NOTICE file corresponding to the section 4 d of == + == the Apache License, Version 2.0, == + == in this case for the Apache Ant distribution. == + ========================================================================= + + This product includes software developed by + The Apache Software Foundation (http://www.apache.org/). + + This product includes also software developed by : + - the W3C consortium (http://www.w3c.org) , + - the SAX project (http://www.saxproject.org) + + Please read the different LICENSE files present in the root directory of + this distribution. + + The names "Ant" and "Apache Software Foundation" must not be used to + endorse or promote products derived from this software without prior + written permission. For written permission, please contact + apache@apache.org. \ No newline at end of file diff --git a/dist/src/main/licenses/bsh.LICENSE b/dist/src/main/licenses/bsh.LICENSE new file mode 100644 index 00000000..491ab11b --- /dev/null +++ b/dist/src/main/licenses/bsh.LICENSE @@ -0,0 +1,34 @@ +From http://www.beanshell.org/license.html + +Dual Licensing: Sun Public License / Gnu Lesser Public License + +BeanShell is dual licensed under both the SPL and LGPL. You may use and +develop BeanShell under either license. + +Please see the Sun Public License +[http://www.sun.com/developers/spl.html] for details. + +About the LGPL + +Note: the LGPL has become the "Lesser Gnu Public License" and BeanShell +has adopted the new version... + +A quick note about the LGPL: + +The LGPL is less restrictive than the ordinary GNU Public License in +that it does not force you to distribute your own applications under +the terms of the GPL. It primarily requires that you include a notice +that you are using the software in your documentation and provide +access to the original source code. It also essentially requires that +if you modify or extend BeanShell itself that you make those changes +available separately, under the terms of either the LGPL or the GPL. +I would ask that you accomodate this by simply sending me your bug +fixes and improvement to allow me to incorporate them into the general +BeanShell release. Please see the LGPL +[http://www.gnu.org/copyleft/lesser.html] for the details. + +If you have a more precise, brief explanation please let me know! +Thanks! - Pat + +Please also feel free to contact me: (Pat Niemeyer pat@pat.net) about +other licensing arrangements. \ No newline at end of file diff --git a/dist/src/main/licenses/commons-cli.LICENSE b/dist/src/main/licenses/commons-cli.LICENSE new file mode 100644 index 00000000..29f81d81 --- /dev/null +++ b/dist/src/main/licenses/commons-cli.LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/dist/src/main/licenses/commons-cli.NOTICE b/dist/src/main/licenses/commons-cli.NOTICE new file mode 100644 index 00000000..7d3f36b2 --- /dev/null +++ b/dist/src/main/licenses/commons-cli.NOTICE @@ -0,0 +1,5 @@ +Apache Commons CLI +Copyright 2002-2006 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). diff --git a/dist/src/main/licenses/commons-codec.LICENSE b/dist/src/main/licenses/commons-codec.LICENSE new file mode 100644 index 00000000..29f81d81 --- /dev/null +++ b/dist/src/main/licenses/commons-codec.LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/dist/src/main/licenses/commons-codec.NOTICE b/dist/src/main/licenses/commons-codec.NOTICE new file mode 100644 index 00000000..5cb73467 --- /dev/null +++ b/dist/src/main/licenses/commons-codec.NOTICE @@ -0,0 +1,5 @@ +Apache Commons Codec +Copyright 2001-2007 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). diff --git a/dist/src/main/licenses/commons-el.LICENSE b/dist/src/main/licenses/commons-el.LICENSE new file mode 100644 index 00000000..29f81d81 --- /dev/null +++ b/dist/src/main/licenses/commons-el.LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/dist/src/main/licenses/commons-el.NOTICE b/dist/src/main/licenses/commons-el.NOTICE new file mode 100644 index 00000000..98c81832 --- /dev/null +++ b/dist/src/main/licenses/commons-el.NOTICE @@ -0,0 +1,5 @@ +Apache Jakarta Commons EL +Copyright 1999-2006 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). diff --git a/dist/src/main/licenses/commons-io.LICENSE b/dist/src/main/licenses/commons-io.LICENSE new file mode 100644 index 00000000..29f81d81 --- /dev/null +++ b/dist/src/main/licenses/commons-io.LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/dist/src/main/licenses/commons-io.NOTICE b/dist/src/main/licenses/commons-io.NOTICE new file mode 100644 index 00000000..5967437a --- /dev/null +++ b/dist/src/main/licenses/commons-io.NOTICE @@ -0,0 +1,5 @@ +Apache Commons IO +Copyright 2002-2008 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). diff --git a/dist/src/main/licenses/commons-lang.LICENSE b/dist/src/main/licenses/commons-lang.LICENSE new file mode 100644 index 00000000..29f81d81 --- /dev/null +++ b/dist/src/main/licenses/commons-lang.LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/dist/src/main/licenses/commons-lang.NOTICE b/dist/src/main/licenses/commons-lang.NOTICE new file mode 100644 index 00000000..b4862deb --- /dev/null +++ b/dist/src/main/licenses/commons-lang.NOTICE @@ -0,0 +1,5 @@ +Apache Commons Lang +Copyright 2001-2008 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). diff --git a/dist/src/main/licenses/commons-logging.LICENSE b/dist/src/main/licenses/commons-logging.LICENSE new file mode 100644 index 00000000..29f81d81 --- /dev/null +++ b/dist/src/main/licenses/commons-logging.LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/dist/src/main/licenses/commons-net.LICENSE b/dist/src/main/licenses/commons-net.LICENSE new file mode 100644 index 00000000..29f81d81 --- /dev/null +++ b/dist/src/main/licenses/commons-net.LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/dist/src/main/licenses/commons-net.NOTICE b/dist/src/main/licenses/commons-net.NOTICE new file mode 100644 index 00000000..5dc8ca6d --- /dev/null +++ b/dist/src/main/licenses/commons-net.NOTICE @@ -0,0 +1,5 @@ +Apache Commons Net +Copyright 2001-2006 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). diff --git a/dist/src/main/licenses/dnsjava-2.0.3.README b/dist/src/main/licenses/dnsjava-2.0.3.README new file mode 100644 index 00000000..479a6f6e --- /dev/null +++ b/dist/src/main/licenses/dnsjava-2.0.3.README @@ -0,0 +1,124 @@ +dnsjava v2.0 + +http://www.xbill.org/dnsjava +http://www.dnsjava.org/ + +Author: + +Brian Wellington (bwelling@xbill.org) +March 12, 2004 + +Overview: + +dnsjava is an implementation of DNS in Java. It supports all defined record +types (including the DNSSEC types), and unknown types. It can be used for +queries, zone transfers, and dynamic updates. It includes a cache which can be +used by clients, and an authoritative only server. It supports TSIG +authenticated messages, partial DNSSEC verification, and EDNS0. It is fully +thread safe. It can be used to replace the native DNS support in Java. + +dnsjava was started as an excuse to learn Java. It was useful for testing new +features in BIND without rewriting the C resolver. It was then cleaned up and +extended in order to be used as a testing framework for DNS interoperability +testing. The high level API and caching resolver were added to make it useful +to a wider audience. The authoritative only server was added as proof of +concept. + + +Getting started: + +Run 'ant' from the toplevel directory to build dnsjava (a Makefile is also +provided, but does not have all of the features of the ant script). JDK 1.4 +or higher is required. + +To compile name service provider support (org.xbill.DNS.spi), run 'ant spi'. + + +Replacing the standard Java DNS functionality: + +Beginning with Java 1.4, service providers can be loaded at runtime. To load +the dnsjava service provider, build it as explained above and set the system +property: + + sun.net.spi.nameservice.provider.1=dns,dnsjava + +This instructs the JVM to use the dnsjava service provide for DNS at the +highest priority. + + +Testing dnsjava: + +Matt Rutherford contributed a number of unit +tests, which are in the tests subdirectory. The hierarchy under tests +mirrors the org.xbill.DNS classes. To build the unit tests, run +'ant compile_tests', and to run then, run 'ant run_tests'. The tests require +JUnit (http://www.junit.org) to be installed. + +Some high-level test programs are in org/xbill/DNS/tests. + + +Limitations: + +There's no standard way to determine what the local nameserver or DNS search +path is at runtime from within the JVM. dnsjava attempts several methods +until one succeeds. + + - The properties 'dns.server' and 'dns.search' (comma delimited lists) are + checked. The servers can either be IP addresses or hostnames (which are + resolved using Java's built in DNS support). + - The sun.net.dns.ResolverConfiguration class is queried. + - On Unix, /etc/resolv.conf is parsed. + - On Windows, ipconfig/winipcfg is called and its output parsed. This may + fail for non-English versions on Windows. + - As a last resort, "localhost" is used as the nameserver, and the search + path is empty. + +The underlying platform must use an ASCII encoding of characters. This means +that dnsjava will not work on OS/390, for example. + + +Additional documentation: + +Javadoc documentation is provided in the doc/ subdirectory of binary +distributions, and can be built with 'ant docs'. + + +License: + +dnsjava is placed under the BSD license. Several files are also under +additional licenses; see the individual files for details. + +Copyright (c) 1999-2005, Brian Wellington +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the dnsjava project nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Final notes: + + - Thanks to Network Associates, Inc. for sponsoring some of the original + dnsjava work in 1999-2000. + + - Thanks to Nominum, Inc. for sponsoring some work on dnsjava from 2000 to + the present. diff --git a/dist/src/main/licenses/fastutil-5.0.7.LICENSE b/dist/src/main/licenses/fastutil-5.0.7.LICENSE new file mode 100644 index 00000000..b87303c3 --- /dev/null +++ b/dist/src/main/licenses/fastutil-5.0.7.LICENSE @@ -0,0 +1,165 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. diff --git a/dist/src/main/licenses/itext.LICENSE b/dist/src/main/licenses/itext.LICENSE new file mode 100644 index 00000000..b3486200 --- /dev/null +++ b/dist/src/main/licenses/itext.LICENSE @@ -0,0 +1,24 @@ +From http://www.lowagie.com/iText/download.html + +iText: a Free Java-PDF library by Bruno Lowagie and Paulo Soares + +License Agreement + +iText is published under 2 different licenses: MPL and LGPL. + +If you are a new user of iText, MPL is recommended (without the LGPL). +MPL is less strict than LGPL. Please read the MPL license agreement +[http://www.lowagie.com/iText/MPL-1.1.txt] before downloading and/or +using iText. + +LGPL is maintained for backward compatibility only. If you choose the +LGPL, you need to mention the MPL as an alternative license. Please +read the LGPL license agreement [http://www.lowagie.com/iText/lgpl.txt] +for more info. + +This library is free and I want it to stay free: you can use it +without paying a fee; you don't need to register anywhere. Only keep +in mind that agreeing with the license is crucial to protect the +software, its developers and its users. This library is distributed +in the hope that it will be useful, but WITHOUT any warranty. If you +don't like free software, don't (ab)use it! diff --git a/dist/src/main/licenses/jasper.LICENSE b/dist/src/main/licenses/jasper.LICENSE new file mode 100644 index 00000000..fbd4284a --- /dev/null +++ b/dist/src/main/licenses/jasper.LICENSE @@ -0,0 +1,204 @@ +jasper-compiler-5.5.15.jar and jasper-runtime-5.5.15.jar are part of Apache +Tomcat, available under the Apache License, Version 2.0. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/dist/src/main/licenses/javaswf.LICENSE b/dist/src/main/licenses/javaswf.LICENSE new file mode 100644 index 00000000..9b948bd7 --- /dev/null +++ b/dist/src/main/licenses/javaswf.LICENSE @@ -0,0 +1,14 @@ +The JavaSWF2-BSD License + +Copyright (c) 2001, David N. Main, All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. The name of the author may not be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/dist/src/main/licenses/je-3.2.44.LICENSE b/dist/src/main/licenses/je-3.2.44.LICENSE new file mode 100644 index 00000000..d2da21a9 --- /dev/null +++ b/dist/src/main/licenses/je-3.2.44.LICENSE @@ -0,0 +1,75 @@ +/*- + * $Id: LICENSE,v 1.10.2.1 2007/02/01 14:49:28 cwl Exp $ + */ + +The following is the license that applies to this copy of the Berkeley +DB Java Edition software. For a license to use the Berkeley DB Java +Edition software under conditions other than those described here, or +to purchase support for this software, please contact Oracle at +berkeleydb-info_us@oracle.com. + +=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= +/* + * Copyright (c) 2002,2007 Oracle. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Redistributions in any form must be accompanied by information on + * how to obtain complete source code for the DB software and any + * accompanying software that uses the DB software. The source code + * must either be included in the distribution or be available for no + * more than the cost of distribution plus a nominal fee, and must be + * freely redistributable under reasonable conditions. For an + * executable file, complete source code means the source code for all + * modules it contains. It does not include source code for modules or + * files that typically accompany the major components of the operating + * system on which the executable file runs. + * + * THIS SOFTWARE IS PROVIDED BY ORACLE ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR + * NON-INFRINGEMENT, ARE DISCLAIMED. IN NO EVENT SHALL ORACLE BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= +/*** + * ASM: a very small and fast Java bytecode manipulation framework + * Copyright (c) 2000-2005 INRIA, France Telecom + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ diff --git a/dist/src/main/licenses/jericho-html-2.3.LICENSE b/dist/src/main/licenses/jericho-html-2.3.LICENSE new file mode 100644 index 00000000..a1a9726c --- /dev/null +++ b/dist/src/main/licenses/jericho-html-2.3.LICENSE @@ -0,0 +1,16 @@ +From http://jerichohtml.sourceforge.net/doc/index.html + +Jericho HTML Parser + +Jericho HTML Parser is a simple but powerful java library allowing +analysis and manipulation of parts of an HTML document, including +some common server-side tags, while reproducing verbatim any +unrecognised or invalid HTML. It also provides high-level HTML form +manipulation functions. + +It is an open source library released under both the Eclipse Public +License (EPL) [http://www.eclipse.org/legal/epl-v10.html] and GNU +Lesser General Public License (LGPL) +[http://www.gnu.org/copyleft/lesser.html]. You are therefore free to +use it in commercial applications subject to the terms detailed in +either one of these licence documents. \ No newline at end of file diff --git a/dist/src/main/licenses/jets3t-0.5.0.LICENSE b/dist/src/main/licenses/jets3t-0.5.0.LICENSE new file mode 100644 index 00000000..29f81d81 --- /dev/null +++ b/dist/src/main/licenses/jets3t-0.5.0.LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/dist/src/main/licenses/jetty.LICENSE b/dist/src/main/licenses/jetty.LICENSE new file mode 100644 index 00000000..29f81d81 --- /dev/null +++ b/dist/src/main/licenses/jetty.LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/dist/src/main/licenses/junit.LICENSE.HTML b/dist/src/main/licenses/junit.LICENSE.HTML new file mode 100644 index 00000000..36aa208d --- /dev/null +++ b/dist/src/main/licenses/junit.LICENSE.HTML @@ -0,0 +1,125 @@ + + + +Common Public License - v 1.0 + + + + + + +

Common Public License - v 1.0 +

+

THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS COMMON PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. +

+

1. DEFINITIONS +

"Contribution" means: + +

    a) in the case of the initial Contributor, the initial code and documentation distributed under this Agreement, and
    +b) in the case of each subsequent Contributor:
+ + +
    i) changes to the Program, and
+ + +
    ii) additions to the Program;
+ + +
    where such changes and/or additions to the Program originate from and are distributed by that particular Contributor. A Contribution 'originates' from a Contributor if it was added to the Program by such Contributor itself or anyone acting on such Contributor's behalf. Contributions do not include additions to the Program which: (i) are separate modules of software distributed in conjunction with the Program under their own license agreement, and (ii) are not derivative works of the Program.
+ +

+

"Contributor" means any person or entity that distributes the Program. +

+

"Licensed Patents " mean patent claims licensable by a Contributor which are necessarily infringed by the use or sale of its Contribution alone or when combined with the Program. +

+

"Program" means the Contributions distributed in accordance with this Agreement. +

+

"Recipient" means anyone who receives the Program under this Agreement, including all Contributors. +

+

2. GRANT OF RIGHTS + +

    a) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, distribute and sublicense the Contribution of such Contributor, if any, and such derivative works, in source code and object code form.
+ + +
+ + +
    b) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed Patents to make, use, sell, offer to sell, import and otherwise transfer the Contribution of such Contributor, if any, in source code and object code form. This patent license shall apply to the combination of the Contribution and the Program if, at the time the Contribution is added by the Contributor, such addition of the Contribution causes such combination to be covered by the Licensed Patents. The patent license shall not apply to any other combinations which include the Contribution. No hardware per se is licensed hereunder.
+ + +
+ + +
    c) Recipient understands that although each Contributor grants the licenses to its Contributions set forth herein, no assurances are provided by any Contributor that the Program does not infringe the patent or other intellectual property rights of any other entity. Each Contributor disclaims any liability to Recipient for claims brought by any other entity based on infringement of intellectual property rights or otherwise. As a condition to exercising the rights and licenses granted hereunder, each Recipient hereby assumes sole responsibility to secure any other intellectual property rights needed, if any. For example, if a third party patent license is required to allow Recipient to distribute the Program, it is Recipient's responsibility to acquire that license before distributing the Program.
+ + +
+ + +
    d) Each Contributor represents that to its knowledge it has sufficient copyright rights in its Contribution, if any, to grant the copyright license set forth in this Agreement.
+ + +
+ +

3. REQUIREMENTS +

A Contributor may choose to distribute the Program in object code form under its own license agreement, provided that: + +

    a) it complies with the terms and conditions of this Agreement; and
+ + +
    b) its license agreement:
+ + +
    i) effectively disclaims on behalf of all Contributors all warranties and conditions, express and implied, including warranties or conditions of title and non-infringement, and implied warranties or conditions of merchantability and fitness for a particular purpose;
+ + +
    ii) effectively excludes on behalf of all Contributors all liability for damages, including direct, indirect, special, incidental and consequential damages, such as lost profits;
+ + +
    iii) states that any provisions which differ from this Agreement are offered by that Contributor alone and not by any other party; and
+ + +
    iv) states that source code for the Program is available from such Contributor, and informs licensees how to obtain it in a reasonable manner on or through a medium customarily used for software exchange.
+ + +
+ +

When the Program is made available in source code form: + +

    a) it must be made available under this Agreement; and
+ + +
    b) a copy of this Agreement must be included with each copy of the Program.
+ +

+

Contributors may not remove or alter any copyright notices contained within the Program. +

+

Each Contributor must identify itself as the originator of its Contribution, if any, in a manner that reasonably allows subsequent Recipients to identify the originator of the Contribution. +

+

4. COMMERCIAL DISTRIBUTION +

Commercial distributors of software may accept certain responsibilities with respect to end users, business partners and the like. While this license is intended to facilitate the commercial use of the Program, the Contributor who includes the Program in a commercial product offering should do so in a manner which does not create potential liability for other Contributors. Therefore, if a Contributor includes the Program in a commercial product offering, such Contributor ("Commercial Contributor") hereby agrees to defend and indemnify every other Contributor ("Indemnified Contributor") against any losses, damages and costs (collectively "Losses") arising from claims, lawsuits and other legal actions brought by a third party against the Indemnified Contributor to the extent caused by the acts or omissions of such Commercial Contributor in connection with its distribution of the Program in a commercial product offering. The obligations in this section do not apply to any claims or Losses relating to any actual or alleged intellectual property infringement. In order to qualify, an Indemnified Contributor must: a) promptly notify the Commercial Contributor in writing of such claim, and b) allow the Commercial Contributor to control, and cooperate with the Commercial Contributor in, the defense and any related settlement negotiations. The Indemnified Contributor may participate in any such claim at its own expense. +

+

For example, a Contributor might include the Program in a commercial product offering, Product X. That Contributor is then a Commercial Contributor. If that Commercial Contributor then makes performance claims, or offers warranties related to Product X, those performance claims and warranties are such Commercial Contributor's responsibility alone. Under this section, the Commercial Contributor would have to defend claims against the other Contributors related to those performance claims and warranties, and if a court requires any other Contributor to pay any damages as a result, the Commercial Contributor must pay those damages. +

+

5. NO WARRANTY +

EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the appropriateness of using and distributing the Program and assumes all risks associated with its exercise of rights under this Agreement, including but not limited to the risks and costs of program errors, compliance with applicable laws, damage to or loss of data, programs or equipment, and unavailability or interruption of operations. +

+

6. DISCLAIMER OF LIABILITY +

EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. +

+

7. GENERAL +

If any provision of this Agreement is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this Agreement, and without further action by the parties hereto, such provision shall be reformed to the minimum extent necessary to make such provision valid and enforceable. +

+

If Recipient institutes patent litigation against a Contributor with respect to a patent applicable to software (including a cross-claim or counterclaim in a lawsuit), then any patent licenses granted by that Contributor to such Recipient under this Agreement shall terminate as of the date such litigation is filed. In addition, if Recipient institutes patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Program itself (excluding combinations of the Program with other software or hardware) infringes such Recipient's patent(s), then such Recipient's rights granted under Section 2(b) shall terminate as of the date such litigation is filed. +

+

All Recipient's rights under this Agreement shall terminate if it fails to comply with any of the material terms or conditions of this Agreement and does not cure such failure in a reasonable period of time after becoming aware of such noncompliance. If all Recipient's rights under this Agreement terminate, Recipient agrees to cease use and distribution of the Program as soon as reasonably practicable. However, Recipient's obligations under this Agreement and any licenses granted by Recipient relating to the Program shall continue and survive. +

+

Everyone is permitted to copy and distribute copies of this Agreement, but in order to avoid inconsistency the Agreement is copyrighted and may only be modified in the following manner. The Agreement Steward reserves the right to publish new versions (including revisions) of this Agreement from time to time. No one other than the Agreement Steward has the right to modify this Agreement. IBM is the initial Agreement Steward. IBM may assign the responsibility to serve as the Agreement Steward to a suitable separate entity. Each new version of the Agreement will be given a distinguishing version number. The Program (including Contributions) may always be distributed subject to the version of the Agreement under which it was received. In addition, after a new version of the Agreement is published, Contributor may elect to distribute the Program (including its Contributions) under the new version. Except as expressly stated in Sections 2(a) and 2(b) above, Recipient receives no rights or licenses to the intellectual property of any Contributor under this Agreement, whether expressly, by implication, estoppel or otherwise. All rights in the Program not expressly granted under this Agreement are reserved. +

+

This Agreement is governed by the laws of the State of New York and the intellectual property laws of the United States of America. No party to this Agreement will bring a legal action under this Agreement more than one year after the cause of action arose. Each party waives its rights to a jury trial in any resulting litigation. +

+

+ + + + \ No newline at end of file diff --git a/dist/src/main/licenses/libidn.LICENSE b/dist/src/main/licenses/libidn.LICENSE new file mode 100644 index 00000000..b87303c3 --- /dev/null +++ b/dist/src/main/licenses/libidn.LICENSE @@ -0,0 +1,165 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. diff --git a/dist/src/main/licenses/mg4j-1.0.1.LICENSE b/dist/src/main/licenses/mg4j-1.0.1.LICENSE new file mode 100644 index 00000000..b87303c3 --- /dev/null +++ b/dist/src/main/licenses/mg4j-1.0.1.LICENSE @@ -0,0 +1,165 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. diff --git a/dist/src/main/licenses/oro-2.0.8.LICENSE b/dist/src/main/licenses/oro-2.0.8.LICENSE new file mode 100644 index 00000000..f571b1b6 --- /dev/null +++ b/dist/src/main/licenses/oro-2.0.8.LICENSE @@ -0,0 +1,53 @@ +/* ==================================================================== + * The Apache Software License, Version 1.1 + * + * Copyright (c) 2000-2002 The Apache Software Foundation. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. The end-user documentation included with the redistribution, + * if any, must include the following acknowledgment: + * "This product includes software developed by the + * Apache Software Foundation (http://www.apache.org/)." + * Alternately, this acknowledgment may appear in the software itself, + * if and wherever such third-party acknowledgments normally appear. + * + * 4. The names "Apache" and "Apache Software Foundation", "Jakarta-Oro" + * must not be used to endorse or promote products derived from this + * software without prior written permission. For written + * permission, please contact apache@apache.org. + * + * 5. Products derived from this software may not be called "Apache" + * or "Jakarta-Oro", nor may "Apache" or "Jakarta-Oro" appear in their + * name, without prior written permission of the Apache Software Foundation. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + */ diff --git a/dist/src/main/licenses/poi.LICENSE b/dist/src/main/licenses/poi.LICENSE new file mode 100644 index 00000000..29f81d81 --- /dev/null +++ b/dist/src/main/licenses/poi.LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/dist/src/main/licenses/poi.NOTICE b/dist/src/main/licenses/poi.NOTICE new file mode 100644 index 00000000..41c966ff --- /dev/null +++ b/dist/src/main/licenses/poi.NOTICE @@ -0,0 +1,35 @@ +Apache POI +Copyright 2001-2007 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + + +Unit testing support is provided by JUnit, under the +Common Public License Version 1.0: + http://www.opensource.org/licenses/cpl.php +See http://www.junit.org/ + +A single data file of the POI component HDGF is based on VSDump, + and is under the GNU General Public Licence version 3 (GPL v3): + http://gplv3.fsf.org/ +Since this is a data file, and has no compiled version (the original + file is distributed in both source and binary versions), there should + be little difference in licencing requirements compared to the ASL. +See http://www.gnome.ru/projects/vsdump_en.html + + +Office Open XML experimental support: + XML Beans - http://xmlbeans.apache.org/ + Apache Licence Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0 + DOM4J - http://www.dom4j.org/ + BSD Licence - http://www.dom4j.org/license.html + Jaxen - http://jaxen.org/ + Apache Style Licence - http://jaxen.org/license.html + OpenXml4J - http://www.openxml4j.org/ + BSD Licence or Apache Licence Version 2.0 - + http://www.openxml4j.org/Licensing/Default.html + Office Open XML ECMA Specification - + http://www.ecma-international.org/publications/standards/Ecma-376.htm + Microsoft Open Specification Promise (OSP) - + http://www.microsoft.com/interop/osp/ diff --git a/dist/src/main/licenses/servlet-4.1.34.LICENSE b/dist/src/main/licenses/servlet-4.1.34.LICENSE new file mode 100644 index 00000000..db8c842c --- /dev/null +++ b/dist/src/main/licenses/servlet-4.1.34.LICENSE @@ -0,0 +1,204 @@ +servlet-4.1.34.jar is part of Apache Tomcat, available under the Apache +License, Version 2.0. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/dist/src/test/java/org/archive/crawler/BasicProfileTest.java b/dist/src/test/java/org/archive/crawler/BasicProfileTest.java new file mode 100644 index 00000000..0c1b6ee5 --- /dev/null +++ b/dist/src/test/java/org/archive/crawler/BasicProfileTest.java @@ -0,0 +1,76 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler; + +import java.io.File; + +import org.apache.commons.io.FileUtils; +import org.archive.spring.PathSharingContext; +import org.archive.util.TmpDirTestCase; +import org.springframework.beans.BeansException; + +/** + * Test all bundled job directories -- that they build, but have + * exactly one validation error (need to enter contact URL). + * + * @contributor pjack + */ +public class BasicProfileTest extends TmpDirTestCase { + + + /** + * Tests the default profile that gets put in the heritrix tarball. + */ + public void testBundledProfiles() throws Exception { + File srcDir = new File("src/main/conf/jobs"); + if (!srcDir.exists()) { + srcDir = new File("dist/src/main/conf/jobs"); + } + if (!srcDir.exists()) { + throw new IllegalStateException("Couldn't find jobs directory"); + } + for (File f: srcDir.listFiles()) { + if (f.isDirectory() && !f.getName().startsWith(".")) { + testProfileDirectory(f); + } + } + } + + protected void testProfileDirectory(File srcDir) throws Exception { + System.out.println("\nNow testing " + srcDir.getName()); + File tmpDir = new File(getTmpDir(), "validatorTest"); + File configDir = new File(tmpDir, srcDir.getName()); + configDir.mkdirs(); + FileUtils.copyDirectory(srcDir, configDir); + + PathSharingContext ac = null; + try { + File config = new File(configDir,"profile-crawler-beans.cxml"); + ac = new PathSharingContext("file:"+config.getAbsolutePath()); + } catch (BeansException be){ + be.printStackTrace(System.err); + } finally { + assertNotNull("profile not buildable",ac); + ac.validate(); + assertEquals("did not get the expected one error",1,ac.getAllErrors().size()); + ac.destroy(); + } + } +} diff --git a/engine/.classpath b/engine/.classpath new file mode 100644 index 00000000..326cf183 --- /dev/null +++ b/engine/.classpath @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/engine/.cvsignore b/engine/.cvsignore new file mode 100644 index 00000000..0209c52e --- /dev/null +++ b/engine/.cvsignore @@ -0,0 +1,13 @@ +disk +jobs +target +.checkstyle +heritrix_dmesg.log +heritrix_out.log +profiles +jsp-compiled +doc +heapdump.* +javacore.* +hs_err_pid* +libsrc diff --git a/engine/.project b/engine/.project new file mode 100644 index 00000000..2e7e4379 --- /dev/null +++ b/engine/.project @@ -0,0 +1,14 @@ + + heritrix + Heritrix is the Internet Archive's open-source, extensible, web-scale, + archival-quality web crawler project. + + + + org.eclipse.jdt.core.javabuilder + + + + org.eclipse.jdt.core.javanature + + \ No newline at end of file diff --git a/engine/pom.xml b/engine/pom.xml new file mode 100644 index 00000000..46c48d47 --- /dev/null +++ b/engine/pom.xml @@ -0,0 +1,150 @@ + + + + org.archive + heritrix + 3.0.0-SNAPSHOT + + 4.0.0 + org.archive.heritrix + engine + jar + Heritrix: 'engine' subproject + + + + true + daily + warn + + + true + never + fail + + agilejava + agilejava.com + http://www.agilejava.com/maven + default + + + + + + + true + daily + warn + + + true + never + fail + + internetarchive + Internet Archive Maven Repository + http://builds.archive.org:8080/maven2 + default + + + + + true + daily + warn + + + true + never + fail + + agilejava + agilejava.com + http://www.agilejava.com/maven + default + + + + maven-restlet + Public online Restlet repository + http://maven.restlet.org + + + + + + org.archive.heritrix + modules + ${project.version} + compile + + + org.mortbay.jetty + jetty + 6.1.14 + compile + + + org.mortbay.jetty + jetty-util + 6.1.14 + compile + + + org.mortbay.jetty + jetty-sslengine + 6.1.14 + compile + + + tomcat + servlet + 4.1.34 + compile + + + org.restlet + org.restlet + 1.1.2 + + + com.noelios.restlet + com.noelios.restlet + 1.1.2 + + + com.noelios.restlet + com.noelios.restlet.ext.jetty + 1.1.2 + + + joda-time + joda-time + 1.6 + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.5 + 1.5 + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + **/TestAll.java + + + + + + diff --git a/engine/src/design/credentials.gif b/engine/src/design/credentials.gif new file mode 100644 index 0000000000000000000000000000000000000000..9993838a160dab9bc4b148109ad438d82263ae20 GIT binary patch literal 18691 zcmV)5K*_&HNk%w1VNe9{0`mX>@9*yb00960|AdBw|NqX6j*I{R000000000000000 z00000000000000000000A^8LW00000EC2ui08j+*0ssX55XecZy*TU5yZ>M)j$~<` zXsWJk>%MR-&vb3yc&_h!@BhG{a7Zi~kI1BQ$!t2G(5Q4uty-_xtai)odcWYXcuX#v z&*-#z&29$+@VIs;jK6uCK7Mva__cwzs&sy1Tr+zQ4f1!o$SH#>dFX%FCw#14*CE z($mz{*4NnC+S}aS-rwNi;^EHFg(+7?(gvN^7Gu~NYV89`uqI-{{H|23LI$6 z-VuBR6DnNDu%W|;5Ff9+xC6Q=wLVW@nv}aKPLqQcysuZbEr5l$zjnmYrReVR83SG*TC|0CH zu_B}j^(oJ@Xw#})%T}DvAxw!@r73mmRjX60nl0NiZ``?j&0^FmIPBfSKmoU1%($`R z$BN04;q!BCbOgOUZ*RW&DhB*0+YQ>u^_f>5g z^X|K%FMHlCTyOE)$A`~e&b+zv=Wd_fyL~vf=G9Gm|JGeyvvTa+!6(Pv8oK%O=+mpe zRU3$0)OA5?1q&YUY0JIkaK~?2tpEO09sWn(R{oW@-hl`vsNi$${Q#4J3q~m6gcMdd zP=h|WM3#jdcIe@UAbNvg4-bZD;)y7xsA5af9kk+$Fvck3j6$_o5sf(JsN;?Yj>toa zJO(M`kVM+#;|`o8spOJOHtFP(P(~@`lvGw}<&{`wsb!NG09oXhV1_AXY;Sov=9y@w zsU}KguIc8RaK;%?n{w7^=bd=ILua0R_UY%J((DQ7poA7`=q!RBs_3GO8k%UMkVY!$ zn~zRv>7|(RSZSu5cIs(}n|>s_U-2 z_Uh}ezy>Squ*4Q??6JrutL(DO{x<9Ev(QE>?X=WZYwfke9yRN>9N9R5nWbSX?zrTZ zYwo$|rmOC{?6&J}ueXBBt*Etf^zOa*=Bw|%{PyebzxUosfV}f|T5l=>1FZ1E3^(lX z!w^SIY{CQ=jOoFsBrNg99Cz&T#~_CcudWqaJSoP_h^+F;EVu0P%hGB*u*nvqYzoD3 zxSR8_ILo}N&%Oe!tkBg0O*F0oAN}*v!X}M$woEI{bkI+eY^=;i11mMNTj!c}vshEx zb=6akT*OkOWLw$f-v9XHkDI*snvzv>FL)lFBub=YcaUG~|BlL~m= zf8(9@;?WLnIJ9;5ZFaK$Mz_7~+uUAJQ;{8C+op zaY#bk*`a`>I$-`!HaNupeXoj2)S>BiH$nHAv3x-cd}lYtYaPhShhX}@{biPVgTF7#3qU_fr1+(AC+e}C>}C=hJ>LcTR1T+ zj!KI{;iBt~_{K&O?sz%uA_YyzLN_*&kV_1kB{^9|y8-c*x76Ss1;|K57V(dq$)h{1xlp^t8y@79Lr!D zGi_4NrecdJP8mj1eX(t&eQGIEUaHe~T~uT}b=Fa*wbW}hb*3ImD!zUS)I14QszuGJ zR=3L4u8xbURlUXI^MTmW zn)b8?^ek#u%UW)hwzaU0?V?^Q+uGVTsH&Y1Xm88g-u{BNuGy_`hfCaA02i>@psRA1 z{>$9vI`_HIjjnQ$aoorzH?7#su6DP}-R^p~Xu`!zZY306SYY+L=uNMB*UR4a_A6cm z8E=Hj`-=0r_rCbeuYUK-->4O%x+a)G;?RDnFX87i`-M5VDnk8IEB1sPs5esYx8rBRh?M7|56uPc~5xg_I_%R6Q< zja7WCN+Nait*ZMxyx1zORSp8znD~I)`(@vCnHPXYv+GKaxr^kvjaSs5@O} z`q3_hrjE3vcX(+GjxU_{!zSXaDAI*qw0{G&qw=B*(2?qzb;U;F6VJJ1E+4d_uU;HJ ztx0HRCKA}QM&+uPChBXynwCj^AxG=!NY6^QSng2R(JW<0F9^Y2PSMf1RJgJguFE2S zx|snV=MtY8?ME^^(zWo#EYr7t?mj*kHT{2>xLbz)u?JRYW?WzCVIwk{=IkVT$5w> zB`)2FX|u<1&Up?d&C}h+dxCq8A2%{4s}65%r`+rVHD$(6xlWdI^vj2D_bJ^T$**~j z@K|@axlt&2!Gn43cCRMgvsZMImVMI&W&77ZZ|9lG`vOwuN$L?sb!c7_(|n9Lu)}Ck z9!_(eVz0cT8860t$Gvth?|s(GZE-`VUH7V^Jt3LC%W&si^OkHrpO5JHckdkJT;zG( z*^KhhXEgcE*ZA3&iG1IWzw*Z?bO~Xu`QD>4_AQ6B@gpAcYW`>I>GMX=>NA#3Pv83D z9wkxulzS#cdlQ9yaFUjE7dUF! zrcQTPNulO<^%r@6Msw9uQ6N}pNEb<~ByFPCc>8mH0hofe22viFYwsg}%r;TEMo=zD zcmv3T$%brT7Ja!xZg3|@P1tkH_HJ<3c%5f#T_aggmm_WVVd>I^Uf5;l0){zOV_~>2 zWSC%`bb1!Be<_lNXJ{=c7KdgyhCby~beMR{kpcqzL$QGh#TcCt+jpDg^u;LiTIe0fM6;yqL2Rgk8u@>_PAx(7yt!Ojs4h&3i&bGB4Pu{Xa#wY^cYGD zS&PMob2{vLDPwG=@G428U&IY zsnH*R2^^?-9{@t0@qv|v`I?7$bHo0Le)Ok-5Jz#a)|jm~Qu$_|u1A0}h;-c+MX*<8 z3*=|E7juH=c@nswLbrFz2chtjLpS$?oG6zt>61nxoPIM$bXo+G11y3s2ZheMB(^q((r+DQz zXWN9KOvxTj=^Gtd9rXdCBWk5oYMLi;qV2e%?^k?d7k;$FWX<`YG#I89S8(z-dxBO> zvS~~|x0q@QaukYiLV9fJC!xWYXKUyY8HzAG*`6CRsCD5TGwNB;R|t-4Pe zN1ffrfH>DsrT3YCnlWQiqSnBr=ICHFiIh<_nR<3Lok;7Qp(A!b+^)HHT_x zF4P)tCAq83Ad`j(Bf~1K;CeR6`Yjj1t?=os;hL^fHLl-OuGjdi&Ecyf0g&`suX4q% zdF8Is2(Qc`uPG^3+IlngIY4qLG% z!mulHvHO~_8Ve#Et0Ep7jrkwx36YT~nr1+=sxv@FY^gX$Mgv8aN%9evrX17eqfX%~a3 zo_?7d_))cnv9pPBmk>f;Alt7_%UBmH9B2C(=1HQ&u^sSnrK~x(*0Hzr5fq*w7fPG5 zYr7mW+qOjVwzv_uVap%NaUFH}8~TA5;+qw5@xa1JEQYs*d zOS(0S8s#w-H!C1jI-;xjx~yBc0Kv2iMvR*ay3~NUnOhlpE2XHbrK&rn+F`n|%ePpX zADa8Cw%a36`=qv@my`=1@zJdMA(g$M*mAggg5974Oc2#VxOCe8bu zIIE=(>mld<`x48`kH09ob=$4(d%Lx3zWCd;^Xt6E<+%V{iTf*JTiTGkT38poTr(aX1_*|(!x79Je46ge$0)-R91@Q!ykg89qglLWY{d(V#aaB4HQdA#jKet$xj{^~ zyGzA*`?~Y0#v2U5%KF9ITEC~uzU*PZe@u`%tH2j*wCSkF;2E2On!Qp?#)sOyfkC?9 zo2C8|;l+F0zewDaF=@Af9Lk@}#iCrwb4*`^%pi=szQv`AAI}ELf|Y#I3x{QOC#RDwlU$##oEX#@oYQyTd@d zA5jdq+iV@BiOH%A$He8zjRDJ_CCl1O&2qaJVayubY#dcw#qM0F*Q}^zY|B>Z%r@K1 z%*)9UEX`+JAJr_jK8(!>&AK0I#{-SW)bX19{JinI#@V2|XRFP=YaflO(824@r~J{= ze9svD&{P@GJgdJHom}d?xL+&GSL>Ht8^VMtzPxOnB-n-Ag8vejS z-6|%X((JL)g3Ju)d?Mz|%;j9vMr~Y3?TUz8Kuzt*{w%*WgVY~O&ck)pZL!q4#no@T z#1mbV0Nuv|-PLA|(pp_GL(7a{Ezxd`kZB#wjm*}$i@_)D$9xLPUozLN2+i%f)@8ld z{hY{8JIaMDCW0N+Qr*Ujt=H4=!lIEKUGdYLq0@X3(=1KQku1!}JS2|_*@!~c2W-$B zn#{D4$~tSvQtR5V3aC|`)L06Uhb-H;z1m<^+KcVeS`obZ%)`KI6d}FM;!Dzp3)QB* zz^L8WwjIK&z1z|K+KQaOtlbjdk-vX^$l1NX8a&Fry};efAM)Ig?5xp;{wlflJj~M@ zCga`QTQ=3SY|JmM-a>4^D2%zie9QZdn&nyF^-Q)~E8u8*%%t%bcPW}Jz2M$!mvF1k zS8EuVz1f}p*)IIxO2ORZz0d%jxY&)!9sRdsY|F?S*c`5Ugq^Pfz0Mx`yB#gkGVS8X zy}BSSx!)YVRC?m0?YAC!;OktPBreIboZ~niw|`6F;*sJA?BTqN(Dz)>bUeH_9=puS zVq3jCP1<-KrFUJ}jZEW88{-@O<1%gIh+5_T<;~^fapGz2(D*&& zB`)K~9puL?7>EAJVolweGU^+e=q$eIoH5evtlDgDx5>@Xl#a)Ye(5l-=IlJ;Ann5# zed|Bm(4vmRrOwz2hU!?UHK^BU^X7(TXAUd+0j&F$IXUAxoJYs@{Kyl*`(aozIYJ>AtkrJSAG9lw`K@5DeN z^tnw{P}$clvGSgNxdr|5a<0RlF66x~&X^_lnOyIAecg)n_Hx|h-7U3YpR;U#_0chZyK;D#C%zr6buj-H7x6p;_?jb8VNPshdG>Giwq_}%$xi}4vR z@{0fIs^8KC-Tc|V^{}h=+AkhT-uzFl?RgLTmHrO6s_D%`uKwvy+yuk)N!#jooYBIa z{Eh3o>)-3t+~wo@`B81r5FkX5bXI(|@y}gYKrQWiSyjM5%%+~F9d|toj z@27phK*2%6Lc>GEM8!o$tbhT4NJ#)XLLDQfFw9M*&7jUS&df_0CsR+YHBnB|&Q{dW zSJ|Yzzs5VFTU=eq|&Dk>8QtVh-)>G~4OK#Oq z?`+ra%=GhB+~MWiWHe^_dV56Mdwt&M{(b*{fK{A5qm-!2lQC?*cmV|lju{|l7$WH) z@kWp%GY*CU2Pfg1EeI`E#ON^xLx-$1c2jsJi5N`%vgA`HGvG~}IepR8+4GoBph1fY zd}fnRJfTUIE|vDKCR3r9V*uBS$R&;2JE`lEx@hN z48EmXH!Z!kcJ=NGqn7U#F@XgSCS2I?VZ<>1F7^Y7VBTGfB~SL_w=8AMe92~p>~C{s z%5fugu0tC1>9?Pcnl_Ev^;pZCr^bH$S9C#qwRJOA?N!ybRJ(;U?Jc>rz2d{y-ac-n zxpL?_lSf82rn((EU$JW{&ArY3Oy5zoDFk0jV4%*?o&%e1dv(4p#21IbZUhvzC!J25Z&n)KXoA5&7Bn(eI{X%f*q>jh{MUIE& z5b2`dK02`?E<$RfoC`>Nv6PKK^v@v`O=J;+8f*N5Eeefegh8Vm#Biw!b35=y18@B8 z$7ySM71YudDgD@z%9(=vsl)QZVTs0Ss!UVFa6a-+oE_U-NKN3VVJD6l z=!EbVBZcbHNhr-osl3R7b56`7qe}9SuCi0oP~alE(lTb}b#b!g4mJ1Jj1i{-D2c@>$Rh=D?#&BQcQ9Oyj#Zley zQZP2k0xP{!9dy?u6{gG!N4N;c=2?xEzGu?B?X5J>%G_Kqc0fPE(0qI^jb*Mr+YfO|DvHIE_fxP8UYDwPm3z zrZvP?<9_vRwf^m1m+GbU#yM44OMQ6RiEMnS5|Q?XNZg6oZfT^a->%kiYF(WbR{bc4 zTvxd<&)DIsA&qifE0)*DlYTIp72zXt48_!i!G zq=_$iW4u;(eR)e-A8T~}0AqUkak@TUAm^Ef3wGQX3i|glzz_cD%)QgO>C3ATc@*;x zw*B|@nT-FHpW#tnd;BX~4tnvTTxq|y0kByfa$MN*R~51q!be?W77q;dA@a;9Ba>TE z8Y(Bi^!1uTK%4KKp2K9d(xrXpY~*o!Nu?% z3H*=7{&e_5872>iL-fiF$Mzl~dW<05E1M6K_{3Tmv4>EkqIjf8AtY8YCp^TI7QJ|- zD@x;wU(BIfZa9=Mj&TuO6r&p57$zaU@rh;JUxwu98Sb62ZHeJyAN}}8Kn7BfgCt}j z3VBFGCK54e1cV;%C>%N-(TJqaKd(iEj61tw)W z@fTDP3~#asOnLYco70ZQm5!L{JdY#wXRB&z=9=6L1{>-A zc~6{(i-7qmoQD+05fxtRT-cJ)8L!Bwrct7_WNXwzGAHLRiP>0bRh&F`Hme1bJA^@R1-vqsJwb0w-` zorb%{Vr-`DxoBcCI?2D{qpz9etf$Dc!NWa}f)Ej5O8`i#5L$I1|C3fOE~x(6qh`*v zn+x1(@r2VTnrJZY!E9$Ov02{+SApXcu4eoaPu?O)r+@uwahl<;hxR2-d-+>l1qV055ylIHeKlYOLzu#xo$!Vuq?-3UkF$<5QfZ5#l7mIG7I3?~QTXRec(6 zwRNI~n<}U+Yq7~)aR_LO@hRl{5ShM4KC+F+dYyIob9Z0+T((%)4*v4i*iTxHFPFRg z(fR#quJn6qo-!6VRe198Z zVbLn(qMeh(YQ`Fd5Bz9Ib8~^9UGixiYwFa_3?>+nsBagI6(Hk-(;fwNsIjSOVt~`u zR^+p%30&)4^AguPnHDl_{b~d@TD}GjHm#d26=XwO+2#fIv+o#dYcp!rKEZYnOtBZ=AE+XfU$i8W5*GL2bRI?mPPIno9J^dbj4G$+~l}-u5aA8qCW~MV{rR zD!bwNpVjiZ6F^wLk%DO0i-G!Z$_)1jydCj#{Bk+_4*17@y?E%Wv}Ky#Ek+M?>o zuXTL@7s_+f&M0^PYy#-5=EaA$9rw9E*5Df?$pkt69W;gSwe15wed-^{`i;0awAT|g z3~~Q8bB8Epl;OcV;tx&j=y_nW{Hq`xH@ zzeuSLWAnab+NU<^pYoHYMp~zUnL+MitVVjk6!w1>phc9Wu zP=Ub!5kh$izalh3*+{;fi=r;3%ud@-hCFN-1*~pqo*cvNAfDNxhYlg|S;bzvG#; zV#Akl87JVw{=>pQ1VjOBJS`9($xAA#TCCcFADE&$!a_e&EUK#FoVHpaQ@BG?SQSpZ zsTKT0P=pRa9K}`~#3me`Q`@@BW5vw_o-uPhVwAgo34vUMK>5);pD7BoR_%oiRE=1mneOgrx5vwce{ZFa#*Cqpi5HEmDgXRy#Ay zF+%0cBw z!S&O~$Kc2Vs>s{JNHZeI`7lZRLH&HkcI-u%sD^G)C!&Mw1E8`DkVOwJQKO|Mi= z=cF*>+^NibPU-}v=!7%m{;W>zq^Ikowd~wZ@T@oC3{UcWuI>!9?<`OCRHO5}vh-X} z_>8;pj8FQ+%=xTO{Dian%uoL0&i(9902NQ>Y{>vEP@DTt15Hr&L{J57Q288CsdP{Y zT~G+c$_c&D2klA%#ZV4SxC%YY4hm6;V^9$-(dr{n6HUC9n&n6QZhZ$GHp^2MN>9yFf(maI8}-%6*U))Q#&25H@#Cm z{kS~cQ$O{LB>hwVLA|~{9aKYoAvK-RLtRw%2vkOW)F}$nB85~+?Xg0wR80MpzpT?t zHPkt^IZo|VLH*QQ3sq6|Q&P1~Q$1BZP1Sl^RaUiAR~0r`ja4|E)mfTOTa8m({kK}( zRW$9@N77YbWm93jwqGsQF+J9gOjc#}Qf7T4Vtv+SbyR8<)o2wrXT4S`%~tl@R&IS# zZ(T=l3Rfi^S28MBb2UrQ+uY_5i!crr&S(*y8jK$d&)Y+YFBC*tk(F9uYkl6_1U&mT9V_Dc1b=HszZJ>oFQNvI&8T52-r6>TB-d(wXsA=G+Qw{yP7#O zB^0Z#McN`8TlhnX1`@lI(!0{4A1eV#qV3r()k3kA+c^k3O2oWmEJo8SHXQO>G0j=Q z3Pid^$Y8Xz%3IqKK{dC1+P`(&$F;<=oXB zqQH&7kc8VUi`|Kw-43wbHpAU_)ZGf;T|e_(#dKXKg5@fUkuD&3bfzN9bf;&KLEzw`5j>WE8y-uU<6*l{Vm=GCg2C|MgEOo4}9DE z)n5z7!U;ah3g%!84qyiUU;^ym=OtkxG~w+<;r3Hu&b9lkvm7T_O_$sqQsi#=i_PGTiqVkXY2W@ut4j$$dEVk)lUSmKBDtl<2V$+jJ0 zd?VT{&R;IxF)uD6FwS71Ro^kD!7?ri1}j1>#?Lj@zBZPEHx6U^onw!z<6=2u{t?zx z`y3O%!iAu_EwZenrx}@59 znVdX?!;pxw(zPDRn}=3YG)$h^7^&L2z1ph1+Eb3htbM3aQ=uYz zM4tgjMTEU$bdpTYDf=uhHU z`)6TJN6MW=L{kS>L_RNcwQk1YZ?3m@9xHf$pSW5qiN-s7<(_OPu^Xg~I5rd8vv=3_JFXGo4z@~vvK4#Kd;=&@ew zvTkd4c|=mx`Niw%)oaAAYqb95xc*+p zerz|k>!_w_lWrV%fim4{!zCo!J(kY79^P?Upkr!_lymAfQ*8hGYM?5W;F-JC4sA-# zvC@`g*`~BB;Xx)OV`_74LX|M8<>gK(#jktAGt^o+H0j)yFL3@D7g;6;G(kgx@2~;f6w!9^D?CXvo5}vz4jc-ZMl^x1AnlaP%kkMj79kM2g$8?(W;o=DYT}yVKj7 zTY;*-v*^Q4+K*28HrZ!{_>0(6gVH|Ke zj%mzZ?2KkE@OCNjuAT|^Xf&(K3xBu_Z*RoiMwPDI(Kb&8C%=d+7dFXm2ok=bvz$sZ zJ_Kv=kW(we#qXZJM|4g_lYnjI3h~z#V^HyiIR9%fr*mGj^Sd5#=QVRlccC@%>nsoOOxJX;;&e9m^iM|~O3!S@ zzVt;$>_)%yM+a&zQDnA|;|nJeE{VR91DIKc)K;(b$vz6v{_WGHbyAOYO|h#Q&u^|f z)LuvRkm=exbAwin$EM@yt{rKiTW+k~+Hv6$Hg-+6ZXoN3?lIJLJ>z9rLGrZQ(`Fa( zBpe`BrZlH1skFVsZ_q?@DDOt7IvB&9{_zMmyKYZ**`#>M3VCzKb&fuHls8s` zmz|1_MUfY1xiaqwNB9qKO_fg{mIrv+`9^WZRF5arS+1A$KIv*VAqhHWt9_htmUkb2 z&TxkyrC;iwE0~K|V z5cS4h^}tSi9Jg(+uqAhL^7gBBOCbG37YM$U)v?Q=Z5|u$=%H<1ef=g9 z_ju=aimsq>x1IdAJiH$DeyQepb7rOoA9<&;DhLQd z$Z0Rmdh_l-7>XlVnkSm7E8BiS;ieTmRg{g}LA~?YEokWxF+tw3sG}i=KOGd=Tgs}| zD>kd$a=Ts*tQ!~r%4Wcgs}$na!_JiLzOdiad-|*2^ZWikph;LDUXG!}9F#_!U8134 zVPu|Clu;ugW~SyQXQ#;E*4UX@o>pSbDcqyn;p3{Mq^svFZLRHX2q<7`TWclQt1o5n zWh^9RU2rKbbF=farf!z#DBAuPQ(dyu>XQ$hGWR51_x9W}G@Qxc-EtUi4Dz(HbX@E}5j?ygW9b&erGPY+j2Ola{U#&-Oylux#Pdt!tMfT83&#&AsdQ?^wBB z!~!mC_%NNkC-;^i92YU<$TS6uk!<-gV#*X5TN{k|GiY_1LxnDFIEm%Z`z}%an=0%ou0!vx z5I1iWAIAfkfzvyByV3#>s!uT(tsrB)>&nE^?$Z&Ob{eXY z812c(=7E9s{zlB4a)KG;p_QhICuvHK=xA_?u=Gobe8NJgrKK`cx`7*y6iQ&il=IxbhhN72Zb_gD5}aS zo0PTJdh4OH(^UgvwB4%KTyN;!_$|A=g$t3iQJLE=z3wJD?!1rIyRW?O638#Wuw~m0 zw3!Bca8Cmtg)qaPDO_#AyEZ&=NDrGwF~)~k3`WGrW&H6$8$Sjz$#sdmNXIXgyfQQ= zTgNiYWw~rb%J9Z~^DHwPi8Ier={!Wu|Monzl^hX$G*Ca=5VUH^rKxVb(IEYr((O1+ z5!Ctq%F^e1QzKlpnyoDijCMPDeOlF5Z&z)nVzV95e`lKtHq*HQi(e)4VOX}7DSBF; zg7!5itA(}7?XbU1^NP0+dRNG|n11{E8?8I$O>5%`O87W@@*JKX-T1C4ERCMpSmO>X zD(NHZjW?<~j-OkW_S?y3iJgwbLdeAGrb{^~>TvQ7r0g1oUexHJGI=6!aQ=?_gTZ&M z>9)nIm$=>3ALw8|UiJ#S?vfN6&^*y+qb2Lsdv|l)dC^IKe7ete*#$qr0V|X zyE&<7R3QXZ^)eWm2C`;V$ogfYBRA z<$@@~9HDQCQ0gIZShySa*}_Clfuawww~`e8L2F^bVr`^Iu1t9mYfkDz8NtXQEov)& zC4pKq(x}F+h*6I7kOCg_sK-6>v5$WIBOn7Q$UzdakcK=YA`_{|MKZFHj(j8}AIZk2 zMY58Xyd)+wsmV=pvXh?tBq%jG$rsdZi*r0B1xC3xRI-u}bqr4{Gx$Rg#*J3fJwFr?-IT z^CE@9rbfzw;sfCGoC=y(J}{{1n;@uYilQMlF@|RKs(;G2Qmw-1N@x|T99{@L^1<+} z%Gf9u*=jPE(t(_-Bq2@b8rI$!^?-&oRtICJK)p(YuSo(dRWDJ`{=$yXu4xJ^V>9E} zrTw#9{(&ql=SSG|{in1h6jYl|o7uJ+tq{n<>_nGAS*wB;w5!$NW+~v;ixn}S@53ny zmsnH0LT-6$xQ9OJgV5VHv$egw8#jK_r5J{Eqk4mttfpHz=4vdrVFRi}^mRqxUe@``=uCE_4{9t_19nnW>|m^T+0GW zOu+|Bu!C92;P^(EUl6YF94Aar3|pAK=HxIIHEc5w)9b<{?xBc-;ACHSS3fL%NT56|<%YdzvaPjch@QCGz}7L9jzD<0?meDjy*9ou4^jLsvUipOM= zv5)0wz#czTwoPtFj;XAiBzKmK#DytGaf-L?@v6FpqbPLSFrgodIT}_5qDFIvT=TsS zxNhcNbmOd4Mxa!=g}dF8>--fmKNGx=Rp79Y<-yY?RA1tJDf;@dXZ2WyLJ6 zbH?hQy86-Q5s0j3>$z7)IoXJEF0Y?`!YN7Hw68YSv}0>+0WG_Q%{FOPy(HRDAC}$U z9`~kq@sHOw6@kAVwOB0+Zf`>r-e>Zuxy{TV-_rhLhFA*raP@pQ9P%m5{x*WH)s0+& z2O6XpTwcOswQx+^>Z~;!tmPiwSb$Hlyfpi?;QXJ5HP-43v>Z?mE2tofCV?Blr0eI;=JvXiSlEF1f# zP*Cr?k;o2Bl)KT28z$3`5f3i&VtMaUzS?!For}Xu{soH9sGNu!ay9qa;0u3q5?{OJ z&Qs*?EXuP?J5?bX8+zE7gU_5dcl!%HPX6<(kM~u|yens~HPWEQH!t2g>EDmpyu^pJ zU~_cTxh9Bs7gwtEn6K=$KRw&925Yom|H0?yRr`x@J+(awes!zWxWUbC@_lc9G=Bcv z>6f5ROK9or6Mz2VK|lEwAKw6kwC&sUQ6B>O-zL!A!W03$rJBZ}6}Tmr_i>)c;oN6Q z-~is;Xz?Fn71S_5paOPL{R!MRP#m0DoKCgb#x=rq@!4;U;Qu+_#0lDVl|$-$lX7jD z5^2w{B})#<;FXl%@Z}8SEtwF$1OS$wfhFNvFyLQ(Me{^nx>%suVPI@fp%Y346kg00 z;uht7p`~;o5{hAWg`pW@3lX+m{u z2(li|otfzEnc+x|=>a3^A<^xn)O3kd?RC!T`JpI2Rs{xG?-dpDMUYjE)a#H=zcC*c z2B7Ti4mj>#@_pd9jiW}jqx_lSHabTD_R0D|6tFc(NI}n6X@*~I}_ z3w~8OaLx`9OAh&8G7g+Bj^3V4)GzL!Ri>GCMPvIt$q+SVA|_?+7@^Cxq!@%GTV9@9 zW`tS>5Y5?8Ev6wB-sLmSWyRbgDEj4Au%cjE)Hn$r8y2R|ETUqfM_uM!V^RfSMkdU? zipdj8|XWfATajF<|I7YsVXQJt0+!Yx^%;wc)nURU$0C`zn#UkdVCw`(=8S12Z za%XlvS_jJUadrq3!+@#pENwdAS&{xIN$sYW=8ZMx(Eae};M@>Z;vP|5nAJl$WY(ddsz3HDJBQ|Y5wUK(k+8;*KvprKkxnggbW z4^a{0P@YqlUX>)krKKKZw0RtlJZYwmDgizprK+TZ(kBV^YP}gBs4AAIZmKqHo1~^{ z_I;@^pxTpyTB&&>JF+U5bYoq8k2!LjOLk_fkrB39f^9LSvpOp3eA)w|541fXuQjRu zg$XwF-=>VGukj;q{@_>bBTcHKhyH3N?q*soUu?-LzZTBEM&NOYT8i>!Kl-4$-jyuQ z{_5)iUcGYLy*})$JtXsB&@UO~3RV@yPN%jWEIgu_<7jNH3an*1nn9kRPa>L0nkz?w z;Hj2uxHjrSR;sAUA5PlqznticNH!O@REmaEYwU_7SO zt_tl(MWV9Sd*3HY-Rj%b`YYjbWY0cjDmh;U(&XHF?bEJK)V73;w$wc8cYf=vBDt2b*jNR?>Ye&4)&X^qHL&g+~Ae3;yTf@ z_9?=euaipS<&Y&^;Vw9U@5;{H*97K*hNlGY=Wkx91@qiqKG=JDop2_Ex#Z;xdMmSm zE7w(UJ8egGUhoVI8G2f16=Lw6KyJXi@F*(B3iqKwSc?ju#RE5CZ2H`D2JvNps0?&v)&leiE@fK4tVj`k}!muZTSq@L|qXcIhaT6<6YJW1JQIM*4y02Un=a)4kBKI8` zFLKO!FoimDAPNT`gM=Q3SQs;22;Z+UswhjuW+76f?E1OKSGcyZM_Mf1Ft`A0)ux@Ud25aRWBO|XdZ{2Lu ziqOVFWXguD-YVoayKCPvtUhaxF>0tZE$z^{tmLL__{gg+D*p0zwk`^evwT7G-hzIa{-*)imbzG}&G*uo|RHO5RL6rFO=k>h#uV zZcgLx9`F19?l*6Bp|;X5H|;X2X!qVP{%gKqK#(nO{`LblE)>3R}|# z(`6qEW?oBXI&892{WS>Z@n9&n5Joi|yKxjhHWSOU1xt3wWU^&n&kuWXX1^0=cQ$`f zwiSc6XTLFNqsC?@=4lJ(XRo%*h;|^mcFVN3YcLt9c)i>9T%Px(;{;u z2YPCaUi&p|>$DvQHypUIa2xk8mT`dU+;IcbZPzhvno~WF#~rIS3|H76Gk0ujcRZ1I zzj{b_)A4ynDhuCMUQ=o83A0(Q^0E?+3gYOWDyc8tFPP>cepk0Ihc7V4_g&35<<$3x zx~VS1DLfeJUDIffE-8Pjlz(q2p$>$DZx4Xu%wV_fIs5EBf3z-}t)$W;m!2v(k6Td_ zWdgGn3@-EqTK7Z&?;DV|94`!4A1HW3IQ^R^ujQ=z2MoIk``YIVh94 za*ykkr*gBR?NllC;sMPFGWWKvnzI#{$D}=%BU~kQj;Gt1bFHs>Zj+a^QAZY1CS;^Ls7wR-C_dNtZWjQXC|4?+l%GNoVLE?jb@XodDV?>L61Y_wI0ARN zDU;8w?QfMsB-uVMswecZq27{`SBH=B2qJbP>ZPW0V`eaR(h^m3cyK)9zbc#240_cSoyuly-@yms`>E5s)udw0GY;%vc46eJ&88(wa zUV2}0wA-R-Z~S;i@;tANKuNbe3y1|u;ad3FNYn(BI}GqbAO6047C zg15YI9kA`?GJb5SgL5yvMHdmtDV}QSo#OYZGF_h^bT+$q{HU|#R;!O!`i~Iwk+FF~ zpEPBqv+bn#JoBGp)yCn}>saU|Bi4ZggE6 zkjI+(Q3bvvTie?sF5kCvPJ4Kv^DT=%oY%{wO{Xc?hB&8OeT_G~SF&^J4z-RY{@+7i zqhG$`Z!P3sw7(Vg=x=A~H~x$3dE5TCy;N#$1VJt6yO_!&bnWu#ryFToFYsH}TCSh^ z&AV37u{w|jGvvVfRj=*385^peAj6GbRb%h}55ecB}mOu z^HXCHXcM&5^c1y4)a5dXl>X(luVN_{BdyjVKo=LKiDox6j5n2dYt^UrR+dD$1h}I) z_gPb5`eOKH*4DbX8}t~~C=lD(BYK6`cY=Io{k-khTVA*lDp^|Jri+c*?-z<5{C(*=hc_- z?&g@)=B|rJw=&*c+O=ZPtepl540`!&>6&W)P_4VOaqPc02EXgMx$y2;qXI9VIJxh= zp$CV5CVpw(_sXZ&XH{L*ZCAOkXV%u=AkB!o3?sDsTh2T61Uzmt1CNW&z6ebKhd%}V zqo6VD@QH@I$258ey0k?C~g0}54mOAsbjV@bjPEDVst3Nidq zv=s?F&qN>*G;%TYID?T%eemnSLjdoR(nlcIlM=CJO6+mSAg@ac%PXr)FUi=NJkypZ z(b4ZA8_B}{D?G2jbWJ9)-sB1?sk|btIicWVjX9WJ((@}m3JMF&L^F_apgXlQ%e3zb zB@aWcQmbmur`}8oN7E7KRS{i#iM7UEft7~4W0j?CG8dPfq0VQet%A{E11Z*$X|>(f5@FTg7Tj?0s8$GK zc|}Ovbk%jT*>&B0ms)QpGQ3O*R&gcV+x;f5W4 znB0aSo|xi_Exs7zj5Xev7#VPtTI;U8{u=DC zo22nvvCTdk?X=Z?gY324ejDz%<+h^jx$V9i@4WTahwHul{u}VX#jacM!U{he@ofoD zobkpTf84&sA)lP`$}Rsy^2;^fob%4%>D%+rMIW8?K|n7Z_0&~g9h%cwe;xMNL(g3H z+HJp`aoTa;o%i17*4_8ug&)3a;E6vT`Q*7ZUis#oe_m1Mp`V`m>LHWf`s}sez7*`a z{~rAC`S4!+^36a03-ZxlpZ)fcaozp+<)7az`02kN|NLXLU;qC7?_YiY6W{>N7ydv3 z9uR@nGhhN4*g)hh(18(@AnqVIK?`0`brrW35Qb)RVGLi5EhB?%sxNx{bAO28TJp|$qi5NZp6%mO^JeClZ*hD9)g^5p; z;uMJyMJiqqi=41x7P;6(Lu}EDVH6`Az&J)Tp3w_sMB^ISXpbYd5sq~e& zT4lTA9{E_mJoXWgfpp^^1zAW#N{x_*ROBMplt@NC5|S@?V%lVQAMCOL`4 zO?DELPyA#kNr}W!mJ*deJY_0bX~R{v5|%G~Wh`lF!dcc5mmR!iE_rFe_+9oAmW;VH*J+gfaoZ%GbILTQ~bDk5O z=~U-B+1XBaz7wADl&8Xo1%LtE6QB9i=RWz_Pk;UspaB)=KnYqEVpqaGEhNmc4nnc7sRJ{77_mFiTfT2-rF6{}g* m>Q=ehRj+;(tYH=FSjk#evz`^LX;tf5+1ggOzO|(h0028fQKaDj literal 0 HcmV?d00001 diff --git a/engine/src/design/credentials.zargo b/engine/src/design/credentials.zargo new file mode 100644 index 0000000000000000000000000000000000000000..c4d0c955b675e759e45f4220ff4afc87cabd2acd GIT binary patch literal 11239 zcma)i1#BHXw`~|ssxY)+X6BqQGcz+YbHmu+2{SV@bJB3qhMAd};kCL?y5INf-lw-M zOZI5CwQbp+y=E;1X-Ft^Ft|VEn^FKH*gp*UPutkZ)Wp=z#nRBmncmRJ+`cMl-F}4; zxp(tb-Q&Bxf0C38K*d%_y&MDCB;f_U?K)oLR@{U;@#E1|V(BH`rcgWEhoGn9R@deg zi@WbPW}~50K2twrO)ScYh$8u{5|cKAoOr=4LkVOQJvqfki35tr<2V&DP}Rb>C#KjP zE+tT$ZfK+}v40PWD5ty-nUJoI-mAwtlIO=4?l4!XUb?wyDngyeXJEivZp8RuG+}{JNi(#5?pYbPm z-<2LM2lvkin3EIZV4NEECVw@>c`@lMkPY5BuTftW-eAy^*Cd{<88Bv6L?$+Ceow@ z){#9-CcWZwbHAdwOIi-gaFi2CvJfGlSeR)VRi|0Mc-wHk@nARRIh*<+Phr>I_Ly`` zI8UYDu%%dzCwB~_3mYwC%EY0x%))dQsnoy2zFdt@=q4f>yJvKnfzlB_T0%xhnjqxP zuDcv{;NWQ(>bXS&+_$=^-TT{x=AmjDy;yU(9jbi;Ksd>626li1!4yQ8Z>^em2one> z`WOq;-_*liH5lhe*&GXZxOm*min-kpS{G~Z?!52M_w{PuwI9DINW;RVtZ_bpLx6#a z!-9b+{C{sB7kd-?hPs-ONkvL&I_jy3Y2{i~1?D*x1tp0|=_&cq329h(517|eEsa#I z6Kra#rB%me%JwJjFcFwm>cqX0JsK!+n6TCtYRUjkFDf!BDS!YOA!6d#C@noW^L_l1 zXxY#+muVT*sAw8l+UO9WCan-M%b)mQ;mFwX%ANjf=A1wEe~%ac-`ibZ*v8P=S;W%N z+{w_EiQd87)@EK;+kTZ3+xN2O*Nb5>0%?pyMEe9O6{aOOT5d{i8sC@5A)$B+akS+d zDXAI6hhYY?)BX%QM_gqpP(NzX)vJdGVh>3Mm+!4Fh9wfGB{*W#c4B$Mya2p5T(ts^M*jXC&~OrD7(W!wJPU zIgQ;nit)A-+FkD%%1=Zg`!tUV&vZ}Cqfd%SkT-va{M8#@rr-P1Hu25O-FfrmCs<>w z>kSev$zFz@zn|us2N*+~-BJ%OcYF1;p5@TuiuQJcremF&URA(yg|pL(OWNRLM*YyM zgQouN-FHxjj^{^K%iyNJYhl|S&e5~{D(k22<_CKp0%&IH>)8;*A{K2wB{3Fb%{ymg zLc_A7bK)Io_kc^sd3<}3=_)&v+V_`E3#WF;eprTq{R*^${RfS}_X%wV5}AEVJ@XUp zXIr)Vo#Ew)Dt(?%K{x5ClgT`7TzdP4hHs;yt`<0{ay{rf<^40ukoBdk=Y?O$1~ysR zhjV*|bwJ#&5TSP=Yb1IJ9`I-R9-4UL-pQeUha6NSKz`m)q~<%^^0nLZwJZApSe{3; zOQ!x&o-1a13|LO1e$dWHu5m-z5*Q;iAzNvgKU5A2`1%vmsu;NcJCbl@f&&vAe+dP2 z$>>s*VTkTh72%A1v9QNvh}Z_1^~07C{6GVEwuU zsSMwW47jET)6_ep1t0rFVBN|FDubXH_|u#R?4P3ncK*2sFp=1UZ5!;Xrng<*d*zx_ z(_VINDYqTAQ4?k{&b>6^0QQ=pG`UIF6Clw+avT|oyYPKsOneUa0dB2ftIk)LP2TtC z4vHM<`ehNZ8eGzI;Z(D9efjE^>oJ7Ib(<9UnBFI^VycS0>9HGtw^y}2O0BShgE z0Al3drm#9<*CU7Gk$qps;4*aV=Hh4JH4WgKTd2)S0>U#;dPGY)bv#Gn*ZVRfu)#1b z(rlLq5pog%f)A(W8B$B(Z7~O~fdwUoQBhS2stn9AG$#?EJ<77iBAx5@3x=%RBQo}0 zSOyjcIj;81R!XzA8-pdG_6(yd-6(VmHCY6T6}GdVwWFsjOmUwA|esy_3hZ-9+e<~ZlOlf6+k*08iH20+r4CV7QXMc}RB~masJ?y$ zU{hkU`ZPB8l17TqRBxAI(6EKPvKY9S_hMX?D#1Jry(%XdxRLke0@#W`3duE451vCn zu**_zg%xi_YK7&q51(7CZ6mtZy5q9>+uo{+URmkiy$GoY^T!ISMj9b23cVK?<&d(( z+lap%GvI*r;)^BUefv~Sl!MCTB_WPp`nPk^mxMe6tZ2IxT&)=AEv>_nD&+vhK-JpI zNuBcwX+SZ(D`CE{C1x0g9pY2%?lZ5+i2cGlYtT$)Ha~~YviC`VS0!L?u710qsstIF2IIB3kD{$hKWu-=G z`8vXQD$huX9jY6rU(j(?OXE#EfiwQSCtg)fbA_dX7 zuP4FjYy6<$UKQ7GqeRpzLpm{1ttedo`TMWV5Y?HK{e8SJ0Mv6)zL~P8ANx9Z$POF` z6{jp}_LIK)*H``;9iAQlwO#1RK}ce;b(9KQ&d8=wYcEk(Qf+r!#}gmY`Pz>`&GiW7 z4X;Y7*eMOaB{%aWv}@doW1;|YKbXvmFO>Fz>^>uM!5<^*7b8jT#F}$mY8lq$b)f>M+s+{3}38MM&+3bci<^2SsC>1wyxw4M>0u3x%%^W)k*HDF^t+- zn?s`R^P~aY5@S{?@xkbn^%>skOZH-mw#Whk8~LIQH5SeS+Agny>K)k3AszaP8s?HY zD?Y%YsdMhEvzLUo_R0qKa&{$i;kkA@e?8Z{cc27w44p85f~oM8faLjoxY>RjNJE3U ziy<8JvGk?3cca)f=Uo>yvbH4XfOmt}OER}=$JHYYEt-^wkXdTvlCNtXQ?her=PWr> zW0*@siB2z3yK!RloxZ{xs_Y@EDL1 zt09kCWq3KRF4bt`_K1}QL%+vQDthDOFj-HLuX(2oOJ1b(QU``Tk1Lgdvzwlg+}YN7 zx~rwkd#OOEV4xE;UgYjWEh)gWcs&~eXbts=a+s)J9ha{1vc&q*_RUm6!YEh&H= z0Eb!T?c^%eWTb3KjH|DVYr(n#UZgzD6H%3_7NY8K6Xz1P2H@BAK@QJnbVLP7B{tXshfV%WS6eTz5RT z)$}fnwL!YW)Jt&lGjVQz#OvRc5%GB77n^yFwcSj2RrA3W^PwF!E&llUz#mk&Z_}~v zF<(9)28`w&CP^zLGU7K0hd1AVI)p&XnY26vXjJV#3zq{&eg%ow)(=M4BPZh))5t2h z|4wr6L-mVnFhd$!U*?OxL?h52l2rTMB`BL2#iwY!`+)7VhxXg*8v7-fWFFD3zI}SA zXMpg2uI=QP_RK=-l>20yTv74?n1q?qqQ1+o5=gvXFnAq!z+3?E0r?2b7kdf(jvbEe z>x!k`##*De5v&X>cP)ThjoZxt$E{Lu34FE}5rVmgb5mL^63^4wV0>(t{w!+E!xJHC z^9q|n5B)>e#M-1OXj>1(teI%SLOgXjl+hTue~T4a?EFnc5T=O^b$?QJ&WLa{#KFXQ z;On?ANsH;~&ZE91_Um>+Z^b1>XVnZ~wQrTW$^qFU+R}e9ZdcS^GnvVITq*-8XmJED z$Sox0@B%}x>SVPqy6S{G3TM)?cO1saBMac_vRp05)S)0;-C-|H17mvdJDntMSlXv@n<@<#Hk?*TIn;IKBoBk`~ zr<3DF3S>lId&amwud0#~Az>s8iV#C{abQm3ODBgYhTh$ZZOOg72_5&{V?XZr9*3o6 zzCsSu$BAq+rj{l)TuT$f8rhG5AuvO_B-1>6=EJ+cSLR#Zih1}ZU_dD23g|ROM{l1= zH|J1mR%5YrUG<*58k4X!o%9N8Q39HD8Kw+;>@eQiyejh>H5aZ4|6-)YQrd2Bmkv-J zIUHw;<~&4>yT;Zu{=&zX81I)22ltkK%t!&>{dX0R$$9{-f9AOVr~dD#2kGCdK<{B| zd9HizyxRKvbE8KnfnjlqHS(U`)v3Gf0L0zmNFST9cBc(POq2!?EhpV4eO%-xG}TUq zi|g<~0e324SzIE=B&=iLKcjck|G2Z=eYo&I26jL0HVA(1XvTUc+jDQc44dchb$jyp zJwO%)lt0?;-o<8jzDd_$eHQtRr%eVu4ZrFybvLa#Fc`&qcznLT_FUO(^M?-( zT)uJh->EBJ&fc$XXq$hl-Fh9z5|17E`Cg~ryD#duMnD?_%kyU26N42QLb}J!V}!GL zV0WX2zv^(^yl9`3)xCGL{9}(!AGgO$XyB$xBCfl{3s3pgXOgG+;5zF4RQx99eYfH1 z6W05tL-*}RpTonO(3}004|Cnmz$<<=Y4)v((IJb6*u&c&Ki(f@Eqboqv$HydK9dNp zc#jt^zGr(}r>OPXUIz`VUG4vP8x{=Cx*ABFq*Dlv)}OwsfTy)&R#6k)-ZDo)nB}S6$@z3SjMHvB_IRyzQ#@R?MgcZhPJcW- zqL6&qj$+iVdTN05MP_;GYoCTVU1;0&r7_Ug#lIU0=N%wfDv-h~i<6}`Ns3&CZsv#o z?B zTbL=W;n3FjNl7NDdGUy*vp8slWq)-=6_5RM|Z zZ_RH-2N$gEK2caSK6do|ezan}O7iL7rcP!#pgr*8Q{r~;(FtU+ANZDOkNBk3#1W0Pqxpb)37@zN4 zc(mUWHtV^E{frv3u0Gv(p7dN?V|7r($t!h{9j1ZAm^Yv24;`V!a1UJjGvL4sr(?oS znP=}ZmR6bD(PU_$a^kD{O@1D~N_vHlm3@E-CkB9U%V=A)v#FLVGQ$0#1h7Nceg#6UxPaTTBsMFEGiw1>qL>b6 z*;AcG;Rr9us96pj>NIl)VK6}x0GKq_?K+-@UM8TCwwOs$4PcUTDTY!jdr(8=7n%L(QUk$ z6PRw50p*|Yi(lUO#T8ca{|EaujP^quXU@lty zNaH$@Lf8E`k)Jbbva~3cm^8`ZbNXRq#C02%-2(QGs^rE|V}Sk2@X=OH&4FVBvE%{H zalxS`@CrJ89c^rS=-A{novPWZ%Gf#Gj4!c+W?*ct4;-VMad&WbuZqypO(om+A620@SFSlQH475LQrj0!XzDg&I@h_^nyMJ+g zJ67;$fg+Kkt`wHL&*7d5o5b)}b6F6?P}D^4=Ein7u!A!|?IrcMWbRdgewBerO&|i7 zsRF!0Y1InB4(NrV7?LUfd4u5O3*)}!7MjXetN1CMk@f1~)#^)}YZD~^XE+rlG;T+V z*6w(_Q!hm+HrWr;MWx*(% zo|O)O6}ALOxSvO|=}!%2f;UF}Ig$1u^VPYWK1uW|T30Z(%`5tfnE>qSpYUlW0tlm) z9r@v6`;CtTTJP#imnruNL2om zZZuH7%m&pd(M{1D+8`r;&S5*I8=|(0*L^Du3@4lbuGa9eKEof7k~7n$*4+fg?rMN5 z5(TDh4xT~Wn(*ieS8K|@!DAqNPp&m`$py~f&Wb!jHUVP<-*2O80vGlWEl^TybEwA49*q6p+D>LBt_s7rb^NW>+c_ zS+>r$XBH`&gJGjwnk36n!Z76g~v$AvM+s&rv? z<)#SH{IH7&bE=jYRTz~mrV7;75qeZh zoGF$8GyvETedZho&K5oJS52su%j=!5htZO!m?U~hWf!N1o~$S7zqK*UIIAL(B0o6m&YnoBh|mW=}y;P@=uxu!}t21{N>gih;w zccuA4v#drbmdQivdmyoeRjS3x<7*5luVK7a>NuU*;EKDHF>1U*+hNG4?3V;d*u4N*yk;mil%0|jh!ff)8`?ztrqUkNV zGfm`$JQDCaiOcc>|gzfIHO?mQ>u>_8&0ZIq< z22jrb^0Wr;(k>T1tYVIW>p&32+X_c=y#OoeO)c&JdfVTGm{2+ONE_*47{<9R)1Ma1 z($t$~d%{|IWED{)$=SgEig+HK7v#$@5MEoZ9Ly4d1Q<*!<)ggqm(-%+U(ylvK|>uO zjgustHHqU&ZG<&Jq&ILUhBsw5ThCkPajCs1;zV()$TA>j%J*b+b_j?}n+80tHu!x0 z7#Piy)JUi`L#Q>&z)cTAXj2ujdzaXEv?b$Ow$p>%nNE{YbA8>%Zxf)G@yWyOq*6uP55)#XnuE_qrN!X5?Z zN_(91e8-l|lPaD5sLH`8S?hS*AO~zY0AzfqZjjf|h8$;l zZs;9(H|oVV#2GX2@Eufjs{(keu!70t)Gpw|c95byNFn6(Yfo|pEgZ6> zXs8a5Owizez0q0$3h+d;Q}=}`;@^zt6(s?){h}w-WfUrzVTmEvJ#$cz`0s`W({b`lWrKGTGi5Wa8q#R=b*xvL_!TpgsI7z{n z-5Mw}MH+zvP+_ZJVHj=v)JebF#xV|hx7ich8Q~FWPz_W-kTeD2D3YM8iGV%4WIrqn zQZXZ)rhcs|tLBggiV-Cf~|POH{--i@wEg znIOo`T8aS!x8sTPB#H8yc^+|LLXSV#!HpmW4&z;TiHAKtVA_2gTR>R zXGiFh+~uDKNO|^%ro8@fLAbn3)aYnf902`_w;$kKe=CEc6q{%dD&qejBMF8qcckfJ zP8nwFZf1~N#%`F!bLK>I(6e&9+biXy>@e8 zryyA{n5(TaNKZgWK**pSIZ6?Wkt_{@6zUKaV)WxNnjhLSv81w(EX%A zqGwRybx?sUPxWe92S`oBUbQ`v%;}>_!J%W))n>(An|Ntkd10^trc69oa@4aOnZ@@r zpA#4y^Ai=ipw%ofU^veSUf^|DVyiqAp05eQzK$d*G|>LkpHk&wJa`WSbHEzt|DpzfUw=Eu%LfkN7z9F}4WLzs1u6Itsb89U=kB4MO+OrRljaWMC8Rz53E@eN-pkc@j!Yz8?Eq$Xn7n9<9IR zK~{qqX!+7oQ>B_VB66AoE}+F-qou55Rc4@QK&*mV2>?Jjl?lxREPcp&gZi)XlH4s7 zykCRJw%xge^Jt_Ut1ulj73WA=K0g2eIYq@yE*ht;Pr8*>`wQ*~WxBp(F4T54>Dn=X zud)i;v?mIZh+?cZpQ)@OaDI27Tj}uRTDEWKys>P!#X^y*yKIfZpG<21$%-bBU6ryh z7-T2}xqSOyHl`w>x zlFx^apX~{(C@)8A0f^Q3BnC0aqM|i9K^_fHCV|}`75!~ET++N*)f~|*(z{$2a51US z2E28df`Wc*_XRuCa;Vu6Gh~Sf`CY5H^hGtrP_NZ`0P7=5D@c)&IY{JCDBdA4UzSoy zRcdNPW6wsYPNIuMfW@K$9FJI_NCQQUGRi1^Fydv97=;i`)y|l{1{(}44IT+)f;3f< zF~PuWPWS@*E#yA(fzW7EM92iST z^yj1JdYY_wOPQ7Fa?-JK(+r`9ruO?V7ri{0@2w&i(YzlIk*a-kLmM$KHIcbgG>g?Q zVwQu_H)!bkkZ-sAQ|Ii&&WJ{E%!hw)LphhYt0_3pM9XB#x=f0jL1iYid?vewx~jXh z3g9aO*EFhtQ(ibx>7d;ON-KG8Kz1V86C0`ez86{^2q}|biQ>dET-cJg)F~Lvc~F{& zZBSkygd2HicPRgB-OQ^hF7>A%qmYq8m9<$BP$5}TWIU&zkO17Z`uCc+KzUtLd7W>S zHDU{{a&{Qg7&K4cLVeX1M44s_Q7pgfX1lA7gd^NCuZo1^kGpmvgc0qgExRQ$xsiHP zi~{ErI&Hy?+!Fbc9N)dry(ha8rB1zW3AeXfw%cGl8Mk-dhHl*D!MWGr1uAwqapI8j zyC&dNF@yo>A?Xh`0wPtad_qiruB_Zbu7D;F-g6x_*n4Qal{tCIpgHnLNc?R7-K)Z-vTP>dHdJ1$Y+@kd zE|+;v4XyI-?eUIBd}fyGK8Gz0V%>oAYmz{-)T=j`PUjEYl=37ih05ZW0o6+IQ*J%M zCmrC`7#h&mRpU~)MjyRq9LEIw>AecgNRT_qOLY6%$tRXblRR7}`_pi>C@#PzRAwBU zv-%5W|J;ap^%NFoTnJcZK*FJaGSt3yJ;J?YB;KNlVC`s43NKp9KNk-b?I6mr&t&Eg_1^u)&;l0`4@CuQO)2PQ zz@^ePF4~TcA^_yo6JdoPLrJXoGPWH-c2yq{bs5|6Y9M>->AVQ+@Ax(>BR< zEdAAGdpB`k6u>{ZA#9s?LkBouMqgknwKiu5cJu)_e^7EuNZ*5`0?AdCQ_AB>Zwh{v z)HLoC6BJ4T3Vz0l*($iU=1zfCoY>Ht%n$1!lj@4MSDUumG5=Q9rsN3@f*v!h#WlhZ zF2V-aiwRvoGxcj@$!^^*>AVR=nh=8J&U=t>r=6N7&L52qj<^iv z6^`Gpjt|PsndD{6m|Cy~tuAIKR|%l1dh{Ab+RwWHOX`o1NEbU)LnApK#c0T9g)Sy| zT2r)mLGWYK{fzE#5o}kL@zILYleorn(7Zou$8PUG-FBe+HN?rSP99q|z2qr>$BEbd z{1SqRVI0i-itg0W^0x8nGH={W_i-1vD)%;maP5x&Szyja}B!vx+Y^*EnrpHmz@ClT0q$qjM{E&e=Vmq9O8Xwtsko0oc{BH!~! z?m#Q}r2qaQNVu*sz2VHwQ9keY ze|+lb581Nj?_KW=kAK4cC8ic|@!WO(dh)^!_VG{7>(H(uRMsPq;9y|issHx=54sK7%>V!Z literal 0 HcmV?d00001 diff --git a/engine/src/main/assembly/dist.xml b/engine/src/main/assembly/dist.xml new file mode 100644 index 00000000..6156c302 --- /dev/null +++ b/engine/src/main/assembly/dist.xml @@ -0,0 +1,25 @@ + + dist + + dir + + true + + + /lib + + + + + target + / + + heritrix-*.jar + + + + target/site + /docs + + + diff --git a/engine/src/main/java/META-INF/MANIFEST-MF b/engine/src/main/java/META-INF/MANIFEST-MF new file mode 100644 index 00000000..e69de29b diff --git a/engine/src/main/java/org/archive/crawler/Heritrix.java b/engine/src/main/java/org/archive/crawler/Heritrix.java new file mode 100644 index 00000000..dddd56b2 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/Heritrix.java @@ -0,0 +1,437 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler; + +import java.io.BufferedOutputStream; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintStream; +import java.security.KeyStore; +import java.security.MessageDigest; +import java.security.cert.Certificate; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.TimeZone; +import java.util.logging.LogManager; +import java.util.logging.Logger; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.output.TeeOutputStream; +import org.archive.crawler.framework.Engine; +import org.archive.crawler.restlet.EngineApplication; +import org.archive.crawler.restlet.RateLimitGuard; +import org.archive.util.ArchiveUtils; +import org.restlet.Component; +import org.restlet.Guard; +import org.restlet.Server; +import org.restlet.data.ChallengeScheme; +import org.restlet.data.Protocol; + +import sun.security.tools.KeyTool; + + +/** + * Main class for Heritrix crawler. + * + * Heritrix is usually launched by a shell script that backgrounds heritrix + * that redirects all stdout and stderr emitted by heritrix to a log file. So + * that startup messages emitted subsequent to the redirection of stdout and + * stderr show on the console, this class prints usage or startup output + * such as where the web UI can be found, etc., to a STARTLOG that the shell + * script is waiting on. As soon as the shell script sees output in this file, + * it prints its content and breaks out of its wait. + * See ${HERITRIX_HOME}/bin/heritrix. + * + *

Heritrix can also be embedded or launched by webapp initialization or + * by JMX bootstrapping. So far I count 4 methods of instantiation: + *

    + *
  1. From this classes main -- the method usually used;
    2. + *
  2. From the Heritrix UI (The local-instances.jsp) page;
    3. + *
  3. A creation by a JMX agent at the behest of a remote JMX client; and
    4. + *
  4. A container such as tomcat or jboss.
    5. + *
+ * + * @author gojomo + * @author Kristinn Sigurdsson + * @author Stack + */ +public class Heritrix { + private static final String ADHOC_PASSWORD = "password"; + + private static final String ADHOC_KEYSTORE = "adhoc.keystore"; + + @SuppressWarnings("unused") + private static final Logger logger = Logger.getLogger(Heritrix.class.getName()); + + /** Name of configuration directory */ + private static final String CONF = "conf"; + + /** Name of the heritrix properties file */ + private static final String PROPERTIES = "logging.properties"; + + protected Engine engine; + protected Component component; + + /** + * Heritrix start log file. + * + * This file contains standard out produced by this main class for startup + * only. Used by heritrix shell script. Name here MUST match that in the + * bin/heritrix shell script. This is a DEPENDENCY the shell + * wrapper has on this here java heritrix. + */ + private static final String STARTLOG = "heritrix_dmesg.log"; + + + private static void usage(PrintStream out) { + HelpFormatter hf = new HelpFormatter(); + hf.printHelp("Heritrix", options()); + } + + + private static Options options() { + Options options = new Options(); + options.addOption("h", "help", true, "Usage information." ); + options.addOption("j", "jobs-dir", true, "The jobs directory. " + + "Defaults to ./jobs"); + options.addOption("l", "logging-properties", true, + "The full path to the logging properties file " + + "(eg, conf/logging.properties). If present, this file " + + "will be used to configure Java logging. Defaults to " + + "./conf/logging.properties"); + options.addOption("a", "webui-admin", true, "Specifies the " + + "authorization password which must be supplied to " + + "access the webui. Required if launching the webui."); + options.addOption("b", "webui-bind-hosts", true, + "A comma-separated list of hostnames for the " + + "webui to bind to."); + options.addOption("p", "webui-port", true, "The port the webui " + + "should listen on."); + options.addOption("r", "run-job", true, "Specify a ready job or a " + + "profile name to launch at launch. If you specify a profile " + + "name, the profile will first be copied to a new ready job, " + + "and that ready job will be launched."); + options.addOption("s", "ssl-params", true, "Specify a keystore " + + "path, keystore password, and key password for HTTPS use. " + + "Separate with commas, no whitespace."); + return options; + } + + + private static File getDefaultPropertiesFile() { + File confDir = new File(CONF); + File props = new File(confDir, PROPERTIES); + return props; + } + + + private static CommandLine getCommandLine(PrintStream out, String[] args) { + CommandLineParser clp = new GnuParser(); + CommandLine cl; + try { + cl = clp.parse(options(), args); + } catch (ParseException e) { + usage(out); + return null; + } + + if (cl.getArgList().size() != 0) { + usage(out); + return null; + } + + return cl; + } + + /** + * Launches a local Engine and restfgul web interface given the + * command-line options or defaults. + * + * @param args Command line arguments. + * @throws Exception + */ + public static void main(String[] args) + throws Exception { + new Heritrix().instanceMain(args); + } + + public void instanceMain(String[] args) + throws Exception { + // Set some system properties early. + // Can't use class names here without loading them. + String ignoredSchemes = "org.archive.net.UURIFactory.ignored-schemes"; + if (System.getProperty(ignoredSchemes) == null) { + System.setProperty(ignoredSchemes, + "mailto, clsid, res, file, rtsp, about"); + } + + String maxFormSize = "org.mortbay.jetty.Request.maxFormContentSize"; + if (System.getProperty(maxFormSize) == null) { + System.setProperty(maxFormSize, "52428800"); + } + + + BufferedOutputStream startupOutStream = + new BufferedOutputStream( + new FileOutputStream( + new File(getHeritrixHome(), STARTLOG)),16384); + PrintStream startupOut = + new PrintStream( + new TeeOutputStream( + System.out, + startupOutStream)); + + CommandLine cl = getCommandLine(startupOut, args); + if (cl == null) return; + + if (cl.hasOption('h')) { + usage(startupOut); + return ; + } + + // DEFAULTS until changed by cmd-line options + int port = 8443; + Set bindHosts = new HashSet(); + String authLogin = "admin"; + String authPassword; + String keystorePath; + String keystorePassword; + String keyPassword; + File properties = getDefaultPropertiesFile(); + + if (cl.hasOption('a')) { + String aOption = cl.getOptionValue('a'); + int colonIndex = aOption.indexOf(':'); + if(colonIndex>-1) { + authLogin = aOption.substring(0,colonIndex); + authPassword = aOption.substring(colonIndex+1); + } else { + authPassword = aOption; + } + } else { + System.err.println( + "You must specify a password for the web interface using -a."); + System.exit(1); + authPassword = ""; // suppresses uninitialized warning + } + + File jobsDir = null; + if (cl.hasOption('j')) { + jobsDir = new File(cl.getOptionValue('j')); + } else { + jobsDir = new File("./jobs"); + } + + if (cl.hasOption('l')) { + properties = new File(cl.getOptionValue('l')); + } + + if (cl.hasOption('b')) { + String hosts = cl.getOptionValue('b'); + List list; + if("/".equals(hosts)) { + // '/' means all, signified by empty-list + list = new ArrayList(); + } else { + list = Arrays.asList(hosts.split(",")); + } + bindHosts.addAll(list); + } else { + // default: only localhost + bindHosts.add("localhost"); + } + + if (cl.hasOption('p')) { + port = Integer.parseInt(cl.getOptionValue('p')); + } + + // SSL options (possibly none, in which case adhoc keystore + // is created or reused + if(cl.hasOption('s')) { + String[] sslParams = cl.getOptionValue('s').split(","); + keystorePath = sslParams[0]; + keystorePassword = sslParams[1]; + keyPassword = sslParams[2]; + } else { + // use ad hoc keystore, creating if necessary + keystorePath = ADHOC_KEYSTORE; + keystorePassword = ADHOC_PASSWORD; + keyPassword = ADHOC_PASSWORD; + useAdhocKeystore(startupOut); + } + + if (properties.exists()) { + FileInputStream finp = new FileInputStream(properties); + LogManager.getLogManager().readConfiguration(finp); + finp.close(); + } + + // Set timezone here. Would be problematic doing it if we're running + // inside in a container. + TimeZone.setDefault(TimeZone.getTimeZone("GMT")); + + // Start Heritrix. + try { + engine = new Engine(jobsDir); + component = new Component(); + + if(bindHosts.isEmpty()) { + // listen all addresses + setupServer(port, null, keystorePath, keystorePassword, keyPassword); + } else { + // bind only to declared addresses, or just 'localhost' + for(String address : bindHosts) { + setupServer(port, address, keystorePath, keystorePassword, keyPassword); + } + } + component.getClients().add(Protocol.FILE); + Guard guard = new RateLimitGuard(null, + ChallengeScheme.HTTP_DIGEST, "Authentication Required"); + guard.getSecrets().put(authLogin, authPassword.toCharArray()); + guard.setNext(new EngineApplication(engine)); + component.getDefaultHost().attach(guard); + component.start(); + startupOut.println("engine listening at port "+port); + startupOut.println("operator login is '"+authLogin + +"' password '"+authPassword+"'"); + + if (cl.hasOption('r')) { + engine.requestLaunch(cl.getOptionValue('r')); + } + } catch (Exception e) { + // Show any exceptions in STARTLOG. + e.printStackTrace(startupOut); + if (component != null) { + component.stop(); + } + throw e; + } finally { + startupOut.flush(); + // stop writing to side startup file + startupOutStream.close(); + System.out.println("Heritrix version: " + + ArchiveUtils.VERSION); + } + } + + /** + * Perform preparation to use an ad-hoc, created-as-necessary + * certificate/keystore for HTTPS access. A keystore with new + * cert is created if necessary, as adhoc.keystore in the working + * directory. Otherwise, a preexisting adhoc.keystore is read + * and the certificate fingerprint shown to assist in operator + * browser-side verification. + * @param startupOut where to report fingerprint + */ + protected void useAdhocKeystore(PrintStream startupOut) { + try { + File keystoreFile = new File(ADHOC_KEYSTORE); + if(!keystoreFile.exists()) { + String[] args = { + "-keystore",ADHOC_KEYSTORE, + "-storepass",ADHOC_PASSWORD, + "-keypass",ADHOC_PASSWORD, + "-alias","adhoc", + "-genkey","-keyalg","RSA", + "-dname", "CN=Heritrix Ad-Hoc HTTPS Certificate"}; + KeyTool.main(args); + } + + KeyStore keystore = KeyStore.getInstance(KeyStore.getDefaultType()); + InputStream inStream = new ByteArrayInputStream( + FileUtils.readFileToByteArray(keystoreFile)); + keystore.load(inStream, ADHOC_PASSWORD.toCharArray()); + Certificate cert = keystore.getCertificate("adhoc"); + byte[] certBytes = cert.getEncoded(); + byte[] sha1 = MessageDigest.getInstance("SHA1").digest(certBytes); + startupOut.print("Using ad-hoc HTTPS certificate with fingerprint...\nSHA1"); + for(byte b : sha1) { + startupOut.print(String.format(":%02X",b)); + } + startupOut.println("\nVerify in browser before accepting exception."); + } catch (Exception e) { + // fatal, rethrow + throw new RuntimeException(e); + } + } + + /** + * Create an HTTPS restlet Server instance matching the given parameters. + * + * @param port + * @param address + * @param keystorePath + * @param keystorePassword + * @param keyPassword + */ + protected void setupServer(int port, String address, String keystorePath, String keystorePassword, String keyPassword) { + Server server = new Server(Protocol.HTTPS,address,port,null); + component.getServers().add(server); + server.getContext().getParameters().add("keystorePath", keystorePath); + server.getContext().getParameters().add("keystorePassword", keystorePassword); + server.getContext().getParameters().add("keyPassword", keyPassword); + } + + /** + * Exploit -Dheritrix.home if available to us. + * Is current working dir if no heritrix.home property supplied. + * @return Heritrix home directory. + * @throws IOException + */ + protected static File getHeritrixHome() + throws IOException { + File heritrixHome = null; + String home = System.getProperty("heritrix.home"); + if (home != null && home.length() > 0) { + heritrixHome = new File(home); + if (!heritrixHome.exists()) { + throw new IOException("HERITRIX_HOME <" + home + + "> does not exist."); + } + } else { + heritrixHome = new File(new File("").getAbsolutePath()); + } + return heritrixHome; + } + + + public Engine getEngine() { + return engine; + } + + + public Component getComponent() { + return component; + } + +} diff --git a/engine/src/main/java/org/archive/crawler/datamodel/CoreAttributeConstants.java b/engine/src/main/java/org/archive/crawler/datamodel/CoreAttributeConstants.java new file mode 100644 index 00000000..c0c9904b --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/datamodel/CoreAttributeConstants.java @@ -0,0 +1,126 @@ +/* Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * CoreAttributeConstants.java + * Created on Jun 17, 2003 + * + * $Header: /cvsroot/archive-crawler/ArchiveOpenCrawler/src//** + * @author gojomo + * + */ +package org.archive.crawler.datamodel; + +import org.archive.modules.ModuleAttributeConstants; + +/** + * CrawlURI attribute keys used by the core crawler + * classes. + * + * @author gojomo + * + */ +public interface CoreAttributeConstants { + + /** + * Extracted MIME type of fetched content; should be + * set immediately by fetching module if possible + * (rather than waiting for a later analyzer) + */ + public static String A_CONTENT_TYPE = "content-type"; + + /** + * Multiplier of last fetch duration to wait before + * fetching another item of the same class (eg host) + */ + public static String A_DELAY_FACTOR = "delay-factor"; + /** + * Minimum delay before fetching another item of th + * same class (eg host). Even if lastFetchTime*delayFactor + * is less than this, this period will be waited. + */ + public static String A_MINIMUM_DELAY = "minimum-delay"; + + public static String A_RRECORD_SET_LABEL = "dns-records"; + public static String A_DNS_FETCH_TIME = "dns-fetch-time"; + public static String A_DNS_SERVER_IP_LABEL = ModuleAttributeConstants.A_DNS_SERVER_IP_LABEL; + public static String A_FETCH_COMPLETED_TIME = "fetch-completed-time"; + public static String A_HTTP_TRANSACTION = ModuleAttributeConstants.A_HTTP_TRANSACTION; + + public static String A_RUNTIME_EXCEPTION = "runtime-exception"; + public static String A_NONFATAL_ERRORS = "nonfatal-errors"; + + /** shorthand string tokens indicating notable occurences, + * separated by commas */ + public static String A_ANNOTATIONS = "annotations"; + + public static String A_PREREQUISITE_URI = "prerequisite-uri"; + public static String A_DISTANCE_FROM_SEED = "distance-from-seed"; + public static String A_HTML_BASE = "html-base-href"; + public static String A_RETRY_DELAY = "retry-delay"; + + /** + * Define for org.archive.crawler.writer.MirrorWriterProcessor. + */ + public static String A_MIRROR_PATH = "mirror-path"; + + /** + * Key to get credential avatars from A_LIST. + */ + public static final String A_CREDENTIAL_AVATARS_KEY = + "credential-avatars"; + + /** a 'source' (usu. URI) that's inherited by discovered URIs */ + public static String A_SOURCE_TAG = ModuleAttributeConstants.A_SOURCE_TAG; + + /** + * Key to (optional) attribute specifying a list of keys that + * are passed to CandidateURIs that 'descend' (are discovered + * via) this URI. + */ + public static final String A_HERITABLE_KEYS = "heritable"; + + /** flag indicating the containing queue should be retired */ + public static final String A_FORCE_RETIRE = "force-retire"; + + /** key to atribute containing precalculated precedence */ + public static final String A_PRECALC_PRECEDENCE = "precalc-precedence"; + + /** local override of proxy host */ + public static final String A_HTTP_PROXY_HOST = "http-proxy-host"; + /** local override of proxy port */ + public static final String A_HTTP_PROXY_PORT = "http-proxy-port"; + + /** local override of origin bind address */ + public static final String A_HTTP_BIND_ADDRESS = "http-bind-address"; + + /** + * Fetch truncation codes present in {@link CrawlURI} annotations. + * All truncation annotations have a TRUNC_SUFFIX suffix (TODO: + * Make for-sure unique or redo truncation so definitive flag marked + * against {@link CrawlURI}). + */ + public static final String TRUNC_SUFFIX = ModuleAttributeConstants.TRUNC_SUFFIX; + // headerTrunc + public static final String HEADER_TRUNC = ModuleAttributeConstants.HEADER_TRUNC; + // timeTrunc + public static final String TIMER_TRUNC = ModuleAttributeConstants.TIMER_TRUNC; + // lenTrunc + public static final String LENGTH_TRUNC = ModuleAttributeConstants.LENGTH_TRUNC; + + +} diff --git a/engine/src/main/java/org/archive/crawler/datamodel/CrawlURI.java b/engine/src/main/java/org/archive/crawler/datamodel/CrawlURI.java new file mode 100644 index 00000000..aa54d247 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/datamodel/CrawlURI.java @@ -0,0 +1,1775 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.datamodel; + +import static org.archive.crawler.datamodel.CoreAttributeConstants.A_ANNOTATIONS; +import static org.archive.crawler.datamodel.CoreAttributeConstants.A_CREDENTIAL_AVATARS_KEY; +import static org.archive.crawler.datamodel.CoreAttributeConstants.A_DNS_SERVER_IP_LABEL; +import static org.archive.crawler.datamodel.CoreAttributeConstants.A_FETCH_COMPLETED_TIME; +import static org.archive.crawler.datamodel.CoreAttributeConstants.A_FORCE_RETIRE; +import static org.archive.crawler.datamodel.CoreAttributeConstants.A_HERITABLE_KEYS; +import static org.archive.crawler.datamodel.CoreAttributeConstants.A_HTML_BASE; +import static org.archive.crawler.datamodel.CoreAttributeConstants.A_NONFATAL_ERRORS; +import static org.archive.crawler.datamodel.CoreAttributeConstants.A_PREREQUISITE_URI; +import static org.archive.crawler.datamodel.CoreAttributeConstants.A_SOURCE_TAG; +import static org.archive.crawler.datamodel.SchedulingConstants.NORMAL; +import static org.archive.modules.fetcher.FetchStatusCodes.S_BLOCKED_BY_CUSTOM_PROCESSOR; +import static org.archive.modules.fetcher.FetchStatusCodes.S_BLOCKED_BY_USER; +import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED; +import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST; +import static org.archive.modules.fetcher.FetchStatusCodes.S_DEEMED_CHAFF; +import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED; +import static org.archive.modules.fetcher.FetchStatusCodes.S_DELETED_BY_USER; +import static org.archive.modules.fetcher.FetchStatusCodes.S_DNS_SUCCESS; +import static org.archive.modules.fetcher.FetchStatusCodes.S_DOMAIN_PREREQUISITE_FAILURE; +import static org.archive.modules.fetcher.FetchStatusCodes.S_DOMAIN_UNRESOLVABLE; +import static org.archive.modules.fetcher.FetchStatusCodes.S_OTHER_PREREQUISITE_FAILURE; +import static org.archive.modules.fetcher.FetchStatusCodes.S_OUT_OF_SCOPE; +import static org.archive.modules.fetcher.FetchStatusCodes.S_PREREQUISITE_UNSCHEDULABLE_FAILURE; +import static org.archive.modules.fetcher.FetchStatusCodes.S_PROCESSING_THREAD_KILLED; +import static org.archive.modules.fetcher.FetchStatusCodes.S_ROBOTS_PRECLUDED; +import static org.archive.modules.fetcher.FetchStatusCodes.S_ROBOTS_PREREQUISITE_FAILURE; +import static org.archive.modules.fetcher.FetchStatusCodes.S_RUNTIME_EXCEPTION; +import static org.archive.modules.fetcher.FetchStatusCodes.S_SERIOUS_ERROR; +import static org.archive.modules.fetcher.FetchStatusCodes.S_TIMEOUT; +import static org.archive.modules.fetcher.FetchStatusCodes.S_TOO_MANY_EMBED_HOPS; +import static org.archive.modules.fetcher.FetchStatusCodes.S_TOO_MANY_LINK_HOPS; +import static org.archive.modules.fetcher.FetchStatusCodes.S_TOO_MANY_RETRIES; +import static org.archive.modules.fetcher.FetchStatusCodes.S_UNATTEMPTED; +import static org.archive.modules.fetcher.FetchStatusCodes.S_UNFETCHABLE_URI; + +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.PrintWriter; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; + +import org.apache.commons.httpclient.HttpMethod; +import org.apache.commons.httpclient.HttpStatus; +import org.apache.commons.httpclient.URIException; +import org.apache.commons.httpclient.methods.GetMethod; +import org.apache.commons.httpclient.methods.PostMethod; +import org.archive.modules.ModuleAttributeConstants; +import org.archive.modules.ProcessorURI; +import org.archive.modules.credential.CredentialAvatar; +import org.archive.modules.credential.Rfc2617Credential; +import org.archive.modules.extractor.HTMLLinkContext; +import org.archive.modules.extractor.Hop; +import org.archive.modules.extractor.Link; +import org.archive.modules.extractor.LinkContext; +import org.archive.net.UURI; +import org.archive.net.UURIFactory; +import org.archive.spring.OverlayContext; +import org.archive.spring.OverlayMapsSource; +import org.archive.util.ArchiveUtils; +import org.archive.util.Base32; +import org.archive.util.MultiReporter; +import org.archive.util.Recorder; + + +/** + * Represents a candidate URI and the associated state it + * collects as it is crawled. + * + *

Core state is in instance variables but a flexible + * attribute list is also available. Use this 'bucket' to carry + * custom processing extracted data and state across CrawlURI + * processing. See the {@link #putString(String, String)}, + * {@link #getString(String)}, etc. + * + * @author Gordon Mohr + */ +public class CrawlURI +implements ProcessorURI, MultiReporter, Serializable, OverlayContext { + private static final long serialVersionUID = 3L; + + public static final int UNCALCULATED = -1; + + /** + * The URI being crawled. It's transient to save space when storing to BDB. + */ + private transient UURI uuri; + + + /** Seed status */ + private boolean isSeed = false; + + + /** String of letters indicating how this URI was reached from a seed. + * 

+     * P precondition
+     * R redirection
+     * E embedded (as frame, src, link, codebase, etc.)
+     * X speculative embed (as from javascript, some alternate-format extractors
+     * L link
+ * For example LLLE (an embedded image on a page 3 links from seed). + */ + private String pathFromSeed; + + /** + * Where this URI was (presently) discovered. . Transient to allow + * more efficient custom serialization + */ + private transient UURI via; + + /** + * Context of URI's discovery, as per the 'context' in Link + */ + private LinkContext viaContext; + + + private int schedulingDirective = NORMAL; + + + /** + * Frontier/Scheduler lifecycle info. + * This is an identifier set by the Frontier for its + * purposes. Usually its the name of the Frontier queue + * this URI gets queued to. Values can be host + port + * or IP, etc. + */ + private String classKey; + + /** assigned precedence */ + private int precedence; + + // Processing progress + private int fetchStatus = 0; // default to unattempted + private int deferrals = 0; // count of postponements for prerequisites + private int fetchAttempts = 0; // the number of fetch attempts that have been made + transient private int threadNumber; + + // User agent to masquerade as when crawling this URI. If null, globals should be used + private String userAgent = null; + + // From header + // TODO: This and user-agent really belong in FetchHTTP +// private transient String from = null; + + // Once a link extractor has finished processing this curi this will be + // set as true + transient private boolean linkExtractorFinished = false; + + /** + * Protection against outlink overflow. + * Change value by setting alternate maximum in heritrix.properties. + */ + public static final int MAX_OUTLINKS = Integer. + parseInt(System.getProperty(CrawlURI.class.getName() + ".maxOutLinks", + "6000")); + + transient private int discardedOutlinks = 0; + +//////////////////////////////////////////////////////////////////// + private long contentSize = UNCALCULATED; + private long contentLength = UNCALCULATED; + + + /** + * Flexible dynamic attributes list. + *

+ * The attribute list is a flexible map of key/value pairs for storing + * status of this URI for use by other processors. By convention the + * attribute list is keyed by constants found in the + * {@link CoreAttributeConstants} interface. Use this list to carry + * data or state produced by custom processors rather change the + * classes {@link CrawlURI} or this class, CrawlURI. + * + * Transient to allow more efficient custom serialization. + * + * Package-protected so CrawlURI can access it directly. + */ + transient Map data; + + +// private transient SheetManager manager; +// private transient StateProvider provider; + + + private boolean forceRevisit = false; // even if already visited + + + /** + * Current http recorder. + * + * Gets set upon successful request. Reset at start of processing chain. + */ + private transient Recorder httpRecorder = null; + + /** + * Content type of a successfully fetched URI. + * + * May be null even on successfully fetched URI. + */ + private String contentType = "unknown"; + + /** + * True if this CrawlURI has been deemed a prerequisite by the + * {@link org.archive.crawler.prefetch.PreconditionEnforcer}. + * + * This flag is used at least inside in the precondition enforcer so that + * subsequent prerequisite tests know to let this CrawlURI through because + * its a prerequisite needed by an earlier prerequisite tests (e.g. If + * this is a robots.txt, then the subsequent login credentials prereq + * test must not throw it out because its not a login curi). + */ + private boolean prerequisite = false; + + + transient private FetchType fetchType = FetchType.UNKNOWN; + + transient private HttpMethod method = null; + + /** + * Monotonically increasing number within a crawl; + * useful for tending towards breadth-first ordering. + * Will sometimes be truncated to 48 bits, so behavior + * over 281 trillion instantiated CrawlURIs may be + * buggy + */ + protected long ordinal; + + /** + * Cache of this candidate uuri as a string. + * + * Profiling shows us spending about 1-2% of total elapsed time in + * toString. + */ + private String cachedCrawlURIString = null; + + /** + * Array to hold keys of data members that persist across URI processings. + * Any key mentioned in this list will not be cleared out at the end + * of a pass down the processing chain. + */ + private static final Collection persistentKeys + = new CopyOnWriteArrayList( + new String [] {A_CREDENTIAL_AVATARS_KEY}); + + /** + * A digest (hash, usually SHA1) of retrieved content-body. + * + */ + private byte[] contentDigest = null; + private String contentDigestScheme = null; + + + /** + * Create a new instance of CrawlURI from a {@link UURI}. + * + * @param uuri the UURI to base this CrawlURI on. + */ + public CrawlURI(UURI uuri) { + this.uuri = uuri; + } + + public static CrawlURI fromHopsViaString(String uriHopsViaContext) throws URIException { + UURI u; + String args[] = uriHopsViaContext.split("\\s+"); + u = UURIFactory.getInstance(args[0]); + String pathFromSeed = (args.length > 1)? + args[1].toString() : ""; + UURI via = (args.length > 2 && args[2].length()>1) ? + UURIFactory.getInstance(args[2].toString()): + null; + LinkContext viaContext = (args.length > 3 && args[2].length()>1) ? + new HTMLLinkContext(args[3].toString()): null; + CrawlURI caUri = new CrawlURI(u, pathFromSeed, via, viaContext); + return caUri; + } + + /** + * @param u uuri instance this CrawlURI wraps. + * @param pathFromSeed + * @param via + * @param viaContext + */ + public CrawlURI(UURI u, String pathFromSeed, UURI via, + LinkContext viaContext) { + this.uuri = u; + this.pathFromSeed = pathFromSeed; + this.via = via; + this.viaContext = viaContext; + } + + + /** + * Create a new instance of CrawlURI from a {@link CrawlURI} + * + * @param caUri the CrawlURI to base this CrawlURI on. + * @param o Monotonically increasing number within a crawl. + */ + public CrawlURI(CrawlURI caUri, long o) { + this(caUri.getUURI(), caUri.getPathFromSeed(), caUri.getVia(), + caUri.getViaContext()); + ordinal = o; + setSeed(caUri.isSeed()); + setSchedulingDirective(caUri.getSchedulingDirective()); + this.data = caUri.data; + } + + + /** + * @return Returns the schedulingDirective. + */ + public int getSchedulingDirective() { + return schedulingDirective; + } + + + /** + * @param priority The schedulingDirective to set. + */ + public void setSchedulingDirective(int priority) { + this.schedulingDirective = priority; + } + + + public boolean containsDataKey(String key) { + if (data == null) { + return false; + } + return data.containsKey(key); + } + + + /** + * Takes a status code and converts it into a human readable string. + * + * @param code the status code + * @return a human readable string declaring what the status code is. + */ + public static String fetchStatusCodesToString(int code){ + switch(code){ + // DNS + case S_DNS_SUCCESS : return "DNS-1-OK"; + // HTTP Informational 1xx + case 100 : return "HTTP-100-Info-Continue"; + case 101 : return "HTTP-101-Info-Switching Protocols"; + // HTTP Successful 2xx + case 200 : return "HTTP-200-Success-OK"; + case 201 : return "HTTP-201-Success-Created"; + case 202 : return "HTTP-202-Success-Accepted"; + case 203 : return "HTTP-203-Success-Non-Authoritative"; + case 204 : return "HTTP-204-Success-No Content "; + case 205 : return "HTTP-205-Success-Reset Content"; + case 206 : return "HTTP-206-Success-Partial Content"; + // HTTP Redirection 3xx + case 300 : return "HTTP-300-Redirect-Multiple Choices"; + case 301 : return "HTTP-301-Redirect-Moved Permanently"; + case 302 : return "HTTP-302-Redirect-Found"; + case 303 : return "HTTP-303-Redirect-See Other"; + case 304 : return "HTTP-304-Redirect-Not Modified"; + case 305 : return "HTTP-305-Redirect-Use Proxy"; + case 307 : return "HTTP-307-Redirect-Temporary Redirect"; + // HTTP Client Error 4xx + case 400 : return "HTTP-400-ClientErr-Bad Request"; + case 401 : return "HTTP-401-ClientErr-Unauthorized"; + case 402 : return "HTTP-402-ClientErr-Payment Required"; + case 403 : return "HTTP-403-ClientErr-Forbidden"; + case 404 : return "HTTP-404-ClientErr-Not Found"; + case 405 : return "HTTP-405-ClientErr-Method Not Allowed"; + case 407 : return "HTTP-406-ClientErr-Not Acceptable"; + case 408 : return "HTTP-407-ClientErr-Proxy Authentication Required"; + case 409 : return "HTTP-408-ClientErr-Request Timeout"; + case 410 : return "HTTP-409-ClientErr-Conflict"; + case 406 : return "HTTP-410-ClientErr-Gone"; + case 411 : return "HTTP-411-ClientErr-Length Required"; + case 412 : return "HTTP-412-ClientErr-Precondition Failed"; + case 413 : return "HTTP-413-ClientErr-Request Entity Too Large"; + case 414 : return "HTTP-414-ClientErr-Request-URI Too Long"; + case 415 : return "HTTP-415-ClientErr-Unsupported Media Type"; + case 416 : return "HTTP-416-ClientErr-Requested Range Not Satisfiable"; + case 417 : return "HTTP-417-ClientErr-Expectation Failed"; + // HTTP Server Error 5xx + case 500 : return "HTTP-500-ServerErr-Internal Server Error"; + case 501 : return "HTTP-501-ServerErr-Not Implemented"; + case 502 : return "HTTP-502-ServerErr-Bad Gateway"; + case 503 : return "HTTP-503-ServerErr-Service Unavailable"; + case 504 : return "HTTP-504-ServerErr-Gateway Timeout"; + case 505 : return "HTTP-505-ServerErr-HTTP Version Not Supported"; + // Heritrix internal codes (all negative numbers + case S_BLOCKED_BY_USER: + return "Heritrix(" + S_BLOCKED_BY_USER + ")-Blocked by user"; + case S_BLOCKED_BY_CUSTOM_PROCESSOR: + return "Heritrix(" + S_BLOCKED_BY_CUSTOM_PROCESSOR + + ")-Blocked by custom prefetch processor"; + case S_DELETED_BY_USER: + return "Heritrix(" + S_DELETED_BY_USER + ")-Deleted by user"; + case S_CONNECT_FAILED: + return "Heritrix(" + S_CONNECT_FAILED + ")-Connection failed"; + case S_CONNECT_LOST: + return "Heritrix(" + S_CONNECT_LOST + ")-Connection lost"; + case S_DEEMED_CHAFF: + return "Heritrix(" + S_DEEMED_CHAFF + ")-Deemed chaff"; + case S_DEFERRED: + return "Heritrix(" + S_DEFERRED + ")-Deferred"; + case S_DOMAIN_UNRESOLVABLE: + return "Heritrix(" + S_DOMAIN_UNRESOLVABLE + + ")-Domain unresolvable"; + case S_OUT_OF_SCOPE: + return "Heritrix(" + S_OUT_OF_SCOPE + ")-Out of scope"; + case S_DOMAIN_PREREQUISITE_FAILURE: + return "Heritrix(" + S_DOMAIN_PREREQUISITE_FAILURE + + ")-Domain prerequisite failure"; + case S_ROBOTS_PREREQUISITE_FAILURE: + return "Heritrix(" + S_ROBOTS_PREREQUISITE_FAILURE + + ")-Robots prerequisite failure"; + case S_OTHER_PREREQUISITE_FAILURE: + return "Heritrix(" + S_OTHER_PREREQUISITE_FAILURE + + ")-Other prerequisite failure"; + case S_PREREQUISITE_UNSCHEDULABLE_FAILURE: + return "Heritrix(" + S_PREREQUISITE_UNSCHEDULABLE_FAILURE + + ")-Prerequisite unschedulable failure"; + case S_ROBOTS_PRECLUDED: + return "Heritrix(" + S_ROBOTS_PRECLUDED + ")-Robots precluded"; + case S_RUNTIME_EXCEPTION: + return "Heritrix(" + S_RUNTIME_EXCEPTION + + ")-Runtime exception"; + case S_SERIOUS_ERROR: + return "Heritrix(" + S_SERIOUS_ERROR + ")-Serious error"; + case S_TIMEOUT: + return "Heritrix(" + S_TIMEOUT + ")-Timeout"; + case S_TOO_MANY_EMBED_HOPS: + return "Heritrix(" + S_TOO_MANY_EMBED_HOPS + + ")-Too many embed hops"; + case S_TOO_MANY_LINK_HOPS: + return "Heritrix(" + S_TOO_MANY_LINK_HOPS + + ")-Too many link hops"; + case S_TOO_MANY_RETRIES: + return "Heritrix(" + S_TOO_MANY_RETRIES + ")-Too many retries"; + case S_UNATTEMPTED: + return "Heritrix(" + S_UNATTEMPTED + ")-Unattempted"; + case S_UNFETCHABLE_URI: + return "Heritrix(" + S_UNFETCHABLE_URI + ")-Unfetchable URI"; + case S_PROCESSING_THREAD_KILLED: + return "Heritrix(" + S_PROCESSING_THREAD_KILLED + ")-" + + "Processing thread killed"; + // Unknown return code + default : return Integer.toString(code); + } + } + + + /** + * Return the overall/fetch status of this CrawlURI for its + * current trip through the processing loop. + * + * @return a value from FetchStatusCodes + */ + public int getFetchStatus(){ + return fetchStatus; + } + + /** + * Set the overall/fetch status of this CrawlURI for + * its current trip through the processing loop. + * + * @param newstatus a value from FetchStatusCodes + */ + public void setFetchStatus(int newstatus){ + fetchStatus = newstatus; + } + + /** + * Get the number of attempts at getting the document referenced by this + * URI. + * + * @return the number of attempts at getting the document referenced by this + * URI. + */ + public int getFetchAttempts() { + return fetchAttempts; + } + + /** + * Increment the number of attempts at getting the document referenced by + * this URI. + * + * @return the number of attempts at getting the document referenced by this + * URI. + */ + public int incrementFetchAttempts() { + // TODO: rename, this is actually processing-loop-attempts + return fetchAttempts++; + } + + /** + * Reset fetchAttempts counter. + */ + public void resetFetchAttempts() { + this.fetchAttempts = 0; + } + + /** + * Reset deferrals counter. + */ + public void resetDeferrals() { + this.deferrals = 0; + } + + + + + /** + * Set a prerequisite for this URI. + *

+ * A prerequisite is a URI that must be crawled before this URI can be + * crawled. + * + * @param link Link to set as prereq. + */ + public void setPrerequisiteUri(Object link) { + getData().put(A_PREREQUISITE_URI, link); + } + + /** + * Get the prerequisite for this URI. + *

+ * A prerequisite is a URI that must be crawled before this URI can be + * crawled. + * + * @return the prerequisite for this URI or null if no prerequisite. + */ + public Object getPrerequisiteUri() { + return getData().get(A_PREREQUISITE_URI); + } + + /** + * @return True if this CrawlURI has a prerequisite. + */ + public boolean hasPrerequisiteUri() { + return containsDataKey(A_PREREQUISITE_URI); + } + + /** + * Returns true if this CrawlURI is a prerequisite. + * + * @return true if this CrawlURI is a prerequisite. + */ + public boolean isPrerequisite() { + return this.prerequisite; + } + + /** + * Set if this CrawlURI is itself a prerequisite URI. + * + * @param prerequisite True if this CrawlURI is itself a prerequiste uri. + */ + public void setPrerequisite(boolean prerequisite) { + this.prerequisite = prerequisite; + } + + /** + * @return This crawl URI as a string wrapped with 'CrawlURI(' + + * ')'. + */ + public String getCrawlURIString() { + if (this.cachedCrawlURIString == null) { + synchronized (this) { + if (this.cachedCrawlURIString == null) { + this.cachedCrawlURIString = + "CrawlURI(" + toString() + ")"; + } + } + } + return this.cachedCrawlURIString; + } + + /** + * Get the content type of this URI. + * + * @return Fetched URIs content type. May be null. + */ + public String getContentType() { + return this.contentType; + } + + /** + * Set a fetched uri's content type. + * + * @param ct Contenttype. + */ + public void setContentType(String ct) { + if (ct == null) { + ct = "unknown"; + } + this.contentType = ct; + } + + /** + * Set the number of the ToeThread responsible for processing this uri. + * + * @param i the ToeThread number. + */ + public void setThreadNumber(int i) { + threadNumber = i; + } + + /** + * Get the number of the ToeThread responsible for processing this uri. + * + * @return the ToeThread number. + */ + public int getThreadNumber() { + return threadNumber; + } + + /** + * Increment the deferral count. + * + */ + public void incrementDeferrals() { + deferrals++; + } + + /** + * Get the deferral count. + * + * @return the deferral count. + */ + public int getDeferrals() { + return deferrals; + } + + /** + * Remove all attributes set on this uri. + *

+ * This methods removes the attribute list. + */ + public void stripToMinimal() { + data = null; + } + + /** + * Get the size in bytes of this URI's recorded content, inclusive + * of things like protocol headers. It is the responsibility of the + * classes which fetch the URI to set this value accordingly -- it is + * not calculated/verified within CrawlURI. + * + * This value is consulted in reporting/logging/writing-decisions. + * + * @see #setContentSize() + * @return contentSize + */ + public long getContentSize(){ + return contentSize; + } + + /** + * Get the annotations set for this uri. + * + * @return the annotations set for this uri. + */ + public Collection getAnnotations() { + @SuppressWarnings("unchecked") + List list = (List)getData().get(A_ANNOTATIONS); + if (list == null) { + list = new ArrayList(); + getData().put(A_ANNOTATIONS, list); + } + return list; + } + + /** + * Get the embeded hop count. + * + * @return the embeded hop count. + */ + public int getEmbedHopCount() { + int embedHops = 0; + for(int i = pathFromSeed.length()-1; i>=0; i--) { + if(pathFromSeed.charAt(i)==Hop.NAVLINK.getHopChar()) { + break; + } + embedHops++; + } + return embedHops; + } + + /** + * Get the link hop count. + * + * @return the link hop count. + */ + public int getLinkHopCount() { + int linkHops = 0; + for(int i = pathFromSeed.length()-1; i>=0; i--) { + if(pathFromSeed.charAt(i)==Hop.NAVLINK.getHopChar()) { + linkHops++; + } + } + return linkHops; + } + + /** + * Get the user agent to use for crawling this URI. + * + * If null the global setting should be used. + * + * @return user agent or null + */ + public String getUserAgent() { + return userAgent; + } + + /** + * Set the user agent to use when crawling this URI. + * + * If not set the global settings should be used. + * + * @param string user agent to use + */ + public void setUserAgent(String string) { + userAgent = string; + } + + + /** + * For completed HTTP transactions, the length of the content-body. + * + * @return For completed HTTP transactions, the length of the content-body. + */ + public long getContentLength() { + if (this.contentLength < 0) { + this.contentLength = (getRecorder() != null)? + getRecorder().getResponseContentLength(): 0; + } + return this.contentLength; + } + + /** + * Get size of data recorded (transferred) + * + * @return recorded data size + */ + public long getRecordedSize() { + return (getRecorder() != null) ? getRecorder() + .getRecordedInput().getSize() + // if unavailable fall back on content-size + : getContentSize(); + } + + /** + * Sets the 'content size' for the URI, which is considered inclusive of all + * of all recorded material (such as protocol headers) or even material + * 'virtually' considered (as in material from a previous fetch + * confirmed unchanged with a server). (In contrast, content-length + * matches the HTTP definition, that of the enclosed content-body.) + * + * Should be set by a fetcher or other processor as soon as the final size + * of recorded content is known. Setting to an artificial/incorrect value + * may affect other reporting/processing. + */ + public void setContentSize(long l) { + contentSize = l; + } + + /** + * If true then a link extractor has already claimed this CrawlURI and + * performed link extraction on the document content. This does not + * preclude other link extractors that may have an interest in this + * CrawlURI from also doing link extraction but default behavior should + * be to not run if link extraction has already been done. + * + *

There is an onus on link extractors to set this flag if they have + * run. + * + *

The only extractor of the default Heritrix set that does not + * respect this flag is + * {@link org.archive.crawler.extractor.ExtractorHTTP}. + * It runs against HTTP headers, not the document content. + * + * @return True if a processor has performed link extraction on this + * CrawlURI + * + * @see #linkExtractorFinished() + */ + public boolean hasBeenLinkExtracted(){ + return linkExtractorFinished; + } + + /** + * Note that link extraction has been performed on this CrawlURI. A processor + * doing link extraction should invoke this method once it has finished it's + * work. It should invoke it even if no links are extracted. It should only + * invoke this method if the link extraction was performed on the document + * body (not the HTTP headers etc.). + * + * @see #hasBeenLinkExtracted() + */ + public void linkExtractorFinished() { + linkExtractorFinished = true; + if(discardedOutlinks>0) { + getAnnotations().add("dol:"+discardedOutlinks); + } + } + + /** + * Notify CrawlURI it is about to be logged; opportunity + * for self-annotation + */ + public void aboutToLog() { + if (fetchAttempts>1) { + getAnnotations().add(fetchAttempts + "t"); + } + } + + /** + * Get the http recorder associated with this uri. + * + * @return Returns the httpRecorder. May be null but its set early in + * FetchHttp so there is an issue if its null. + */ + public Recorder getRecorder() { + return httpRecorder; + } + + /** + * Set the http recorder to be associated with this uri. + * + * @param httpRecorder The httpRecorder to set. + */ + public void setRecorder(Recorder httpRecorder) { + this.httpRecorder = httpRecorder; + } + + /** + * Return true if this is a http transaction. + * + * TODO: Compound this and {@link #isPost()} method so that there is one + * place to go to find out if get http, post http, ftp, dns. + * + * @return True if this is a http transaction. + */ + public boolean isHttpTransaction() { + return method != null; + } + + /** + * Clean up after a run through the processing chain. + * + * Called on the end of processing chain by Frontier#finish. Null out any + * state gathered during processing. + */ + public void processingCleanup() { + this.httpRecorder = null; + this.fetchStatus = S_UNATTEMPTED; + this.setPrerequisite(false); + this.contentSize = UNCALCULATED; + this.contentLength = UNCALCULATED; + // Clear 'links extracted' flag. + this.linkExtractorFinished = false; + // Clean the data map of all but registered permanent members. + this.data = getPersistentDataMap(); + } + + public Map getPersistentDataMap() { + if (data == null) { + return null; + } + Map result = new HashMap(getData()); + Set retain = new HashSet(persistentKeys); + + if (containsDataKey(A_HERITABLE_KEYS)) { + @SuppressWarnings("unchecked") + List heritable = (List)getData().get(A_HERITABLE_KEYS); + retain.addAll(heritable); + } + + result.keySet().retainAll(retain); + return result; + } + + /** + * Make a CrawlURI from the passed CrawlURI. + * + * Its safe to pass a CrawlURI instance. In this case we just return it + * as a result. Otherwise, we create new CrawlURI instance. + * + * @param caUri Candidate URI. + * @param ordinal + * @return A crawlURI made from the passed CrawlURI. + */ +// public static CrawlURI from(CrawlURI caUri, long ordinal) { +// return (caUri instanceof CrawlURI)? +// (CrawlURI)caUri: new CrawlURI(caUri, ordinal); +// } + + + /** + * @return Credential avatars. Null if none set. + */ + public Set getCredentialAvatars() { + @SuppressWarnings("unchecked") + Set r = (Set)getData().get(A_CREDENTIAL_AVATARS_KEY); + if (r == null) { + r = new HashSet(); + getData().put(A_CREDENTIAL_AVATARS_KEY, r); + } + return r; + } + + /** + * @return True if there are avatars attached to this instance. + */ + public boolean hasCredentialAvatars() { + return containsDataKey(A_CREDENTIAL_AVATARS_KEY); + } + + + /** + * Ask this URI if it was a success or not. + * + * Only makes sense to call this method after execution of + * HttpMethod#execute. Regard any status larger then 0 as success + * except for below caveat regarding 401s. Use {@link #is2XXSuccess()} if + * looking for a status code in the 200 range. + * + *

401s caveat: If any rfc2617 credential data present and we got a 401 + * assume it got loaded in FetchHTTP on expectation that we're to go around + * the processing chain again. Report this condition as a failure so we + * get another crack at the processing chain only this time we'll be making + * use of the loaded credential data. + * + * @return True if ths URI has been successfully processed. + * @see #is2XXSuccess() + */ + public boolean isSuccess() { + boolean result = false; + int statusCode = this.fetchStatus; + if (statusCode == HttpStatus.SC_UNAUTHORIZED && + hasRfc2617CredentialAvatar()) { + result = false; + } else { + result = (statusCode > 0); + } + return result; + } + + /** + * @return True if status code is in the 2xx range. + * @see #isSuccess() + */ + public boolean is2XXSuccess() { + return this.fetchStatus >= 200 && this.fetchStatus < 300; + } + + /** + * @return True if we have an rfc2617 payload. + */ + public boolean hasRfc2617CredentialAvatar() { + boolean result = false; + Set avatars = getCredentialAvatars(); + if (avatars != null && avatars.size() > 0) { + for (Iterator i = avatars.iterator(); i.hasNext();) { + if (((CredentialAvatar)i.next()). + match(Rfc2617Credential.class)) { + result = true; + break; + } + } + } + return result; + } + + + /** + * Set the retained content-digest value (usu. SHA1). + * + * @param digestValue + * @deprecated Use {@link #setContentDigest(String scheme, byte[])} + */ + public void setContentDigest(byte[] digestValue) { + setContentDigest("SHA1", digestValue); + } + + public void setContentDigest(final String scheme, + final byte [] digestValue) { + this.contentDigest = digestValue; + this.contentDigestScheme = scheme; + } + + public String getContentDigestSchemeString() { + if (this.contentDigest == null) { + return null; + } + return this.contentDigestScheme + ":" + getContentDigestString(); + } + + /** + * Return the retained content-digest value, if any. + * + * @return Digest value. + */ + public byte[] getContentDigest() { + return contentDigest; + } + + public String getContentDigestString() { + if (this.contentDigest == null) { + return null; + } + return Base32.encode(this.contentDigest); + } + + transient Object holder; + transient Object holderKey; + + /** + * Remember a 'holder' to which some enclosing/queueing + * facility has assigned this CrawlURI + * . + * @param obj + */ + public void setHolder(Object obj) { + holder=obj; + } + + /** + * Return the 'holder' for the convenience of + * an external facility. + * + * @return holder + */ + public Object getHolder() { + return holder; + } + + /** + * Remember a 'holderKey' which some enclosing/queueing + * facility has assigned this CrawlURI + * . + * @param obj + */ + public void setHolderKey(Object obj) { + holderKey=obj; + } + /** + * Return the 'holderKey' for convenience of + * an external facility (Frontier). + * + * @return holderKey + */ + public Object getHolderKey() { + return holderKey; + } + + /** + * Get the ordinal (serial number) assigned at creation. + * + * @return ordinal + */ + public long getOrdinal() { + return ordinal; + } + + + public void setOrdinal(long o) { + this.ordinal = o; + } + + /** spot for an integer cost to be placed by external facility (frontier). + * cost is truncated to 8 bits at times, so should not exceed 255 */ + int holderCost = UNCALCULATED; + /** + * Return the 'holderCost' for convenience of external facility (frontier) + * @return value of holderCost + */ + public int getHolderCost() { + return holderCost; + } + + /** + * Remember a 'holderCost' which some enclosing/queueing + * facility has assigned this CrawlURI + * @param cost value to remember + */ + public void setHolderCost(int cost) { + holderCost = cost; + } + + /** + * All discovered outbound Links (navlinks, embeds, etc.) + * Can either contain Link instances or CrawlURI instances, or both. + * The LinksScoper processor converts Link instances in this collection + * to CrawlURI instances. + */ + transient Collection outLinks = new HashSet(); + + + transient Collection outCandidates = new HashSet(); + + /** + * Returns discovered links. The returned collection might be empty if + * no links were discovered, or if something like LinksScoper promoted + * the links to CrawlURIs. + * + * @return Collection of all discovered outbound Links + */ + public Collection getOutLinks() { + return outLinks; +// return Transform.subclasses(outLinks, Link.class); + } + + /** + * Returns discovered candidate URIs. The returned collection will be + * emtpy until something like LinksScoper promotes discovered Links + * into CrawlURIs. + * + * @return Collection of candidate URIs + */ + public Collection getOutCandidates() { + return outCandidates; + } + + + + + + + /** + * Set the (HTML) Base URI used for derelativizing internal URIs. + * + * @param baseHref String base href to use + * @throws URIException if supplied string cannot be interpreted as URI + */ + public void setBaseURI(String baseHref) throws URIException { + getData().put(A_HTML_BASE, UURIFactory.getInstance(baseHref)); + } + + /** + * Get the (HTML) Base URI used for derelativizing internal URIs. + * + * @return UURI base URI previously set + */ + public UURI getBaseURI() { + if (!containsDataKey(A_HTML_BASE)) { + return getUURI(); + } + return (UURI)getData().get(A_HTML_BASE); + } + + /** + * Add the key of items you want to persist across + * processings. + * @param key Key to add. + */ + public static Collection getPersistentDataKeys() { + return persistentKeys; + } + + public void addPersistentDataMapKey(String s) { + if (!persistentKeys.contains(s)) { + addDataPersistentMember(s); + } + } + + /** + * Add the key of data map items you want to persist across + * processings. + * @param key Key to add. + */ + public static void addDataPersistentMember(String key) { + persistentKeys.add(key); + } + + /** + * Remove the key from those data map members persisted. + * @param key Key to remove. + * @return True if list contained the element. + */ + public static boolean removeDataPersistentMember(String key) { + return persistentKeys.remove(key); + } + + /** + * Custom serialization writing an empty 'outLinks' as null. Estimated + * to save ~20 bytes in serialized form. + * + * @param stream + * @throws IOException + */ + private void writeObject(ObjectOutputStream stream) throws IOException { + stream.defaultWriteObject(); + stream.writeUTF(uuri.toCustomString()); // + stream.writeObject((via == null) ? null : via.getURI()); + stream.writeObject((data==null || data.isEmpty()) ? null : data); + stream.writeObject((outLinks.isEmpty()) ? null : outLinks); + stream.writeObject((outCandidates.isEmpty()) ? null : outCandidates); + } + + /** + * Custom deserialization recreating empty HashSet from null in 'outLinks' + * slot. + * + * @param stream + * @throws IOException + * @throws ClassNotFoundException + */ + private void readObject(ObjectInputStream stream) throws IOException, + ClassNotFoundException { + stream.defaultReadObject(); + uuri = readUuri(stream.readUTF()); + via = readUuri((String)stream.readObject()); + @SuppressWarnings("unchecked") + Map temp = (Map)stream.readObject(); + this.data = temp; + + @SuppressWarnings("unchecked") + HashSet ol = (HashSet) stream.readObject(); + outLinks = (ol == null) ? new HashSet() : ol; + @SuppressWarnings("unchecked") + HashSet oc = (HashSet)stream.readObject(); + outCandidates = (oc == null) ? new HashSet() : oc; + } + + + + /** + * Read a UURI from a String, handling a null or URIException + * + * @param u String or null from which to create UURI + * @return the best UURI instance creatable + */ + protected UURI readUuri(String u) { + if (u == null) { + return null; + } + try { + return UURIFactory.getInstance(u); + } catch (URIException ux) { + // simply continue to next try + } + try { + // try adding an junk scheme + return UURIFactory.getInstance("invalid:" + u); + } catch (URIException ux) { + ux.printStackTrace(); + // ignored; method continues + } + try { + // return total junk + return UURIFactory.getInstance("invalid:"); + } catch (URIException e) { + e.printStackTrace(); + return null; + } + } + + + + public String getDNSServerIPLabel() { + if (data == null) { + return null; + } else { + return (String)data.get(A_DNS_SERVER_IP_LABEL); + } + } + + public long getFetchBeginTime() { + if (containsDataKey(ModuleAttributeConstants.A_FETCH_BEGAN_TIME)) { + return (Long)getData().get(ModuleAttributeConstants.A_FETCH_BEGAN_TIME); + } else { + return 1L; + } + } + + public long getFetchCompletedTime() { + if (containsDataKey(A_FETCH_COMPLETED_TIME)) { + return (Long)getData().get(A_FETCH_COMPLETED_TIME); + } else { + return 0L; + } + } + + public long getFetchDuration() { + if (!containsDataKey(A_FETCH_COMPLETED_TIME)) { + return -1; + } + + long completedTime = getFetchCompletedTime(); + long beganTime = getFetchBeginTime(); + return completedTime - beganTime; + } + + public FetchType getFetchType() { + return fetchType; + } + + public Collection getNonFatalFailures() { + @SuppressWarnings("unchecked") + List list = (List)getData().get(A_NONFATAL_ERRORS); + if (list == null) { + list = new ArrayList(); + getData().put(A_NONFATAL_ERRORS, list); + } + + // FIXME: Previous code automatically added annotation when "localized error" + // was added, override collection to implement that? + return list; + } + + + public void setDNSServerIPLabel(String label) { + getData().put(A_DNS_SERVER_IP_LABEL, label); + } + + public void setError(String msg) { + // TODO: Figure out where this is read, if ever. + getData().put("error", msg); + } + + public void setFetchBeginTime(long time) { + getData().put(ModuleAttributeConstants.A_FETCH_BEGAN_TIME, time); + } + + public void setFetchCompletedTime(long time) { + getData().put(A_FETCH_COMPLETED_TIME, time); + } + + public void setFetchType(FetchType type) { + fetchType = type; + } + + public void setHttpMethod(HttpMethod method) { + this.method = method; + if (method instanceof PostMethod) { + fetchType = FetchType.HTTP_POST; + } else if (method instanceof GetMethod) { + fetchType = FetchType.HTTP_GET; + } else { + fetchType = FetchType.UNKNOWN; + } + } + + public void setForceRetire(boolean b) { + getData().put(A_FORCE_RETIRE, b); + } + + public HttpMethod getHttpMethod() { + return method; + } + + public void setBaseURI(UURI base) { + getData().put(A_HTML_BASE, base); + } + + public Map getData() { + if (data == null) { + data = new HashMap(); + } + return data; + } + + /** + * Set the isSeed attribute of this URI. + * @param b Is this URI a seed, true or false. + */ + public void setSeed(boolean b) { + this.isSeed = b; + if (this.isSeed) { + if(pathFromSeed==null) { + this.pathFromSeed = ""; + } +// seeds created on redirect must have a via to be recognized; don't clear +// setVia(null); + } + } + + /** + * @return Whether seeded. + */ + public boolean isSeed() { + return this.isSeed; + } + + /** + * @return UURI + */ + public UURI getUURI() { + return this.uuri; + } + + /** + * @return path (hop-types) from seed + */ + public String getPathFromSeed() { + return this.pathFromSeed; + } + + /** + * @return URI via which this one was discovered + */ + public UURI getVia() { + return this.via; + } + + + public void setVia(UURI via) { + this.via = via; + } + + + /** + * @return CharSequence context in which this one was discovered + */ + public LinkContext getViaContext() { + return this.viaContext; + } + + + /** + * @return True if this CrawlURI was result of a redirect: + * i.e. Its parent URI redirected to here, this URI was what was in + * the 'Location:' or 'Content-Location:' HTTP Header. + */ + public boolean isLocation() { + return this.pathFromSeed != null && this.pathFromSeed.length() > 0 && + this.pathFromSeed.charAt(this.pathFromSeed.length() - 1) == + Hop.REFER.getHopChar(); + } + + +// public void setStateProvider(SheetManager manager) { +// if(this.provider!=null) { +// return; +// } +// this.manager = manager; +//// this.provider = manager.findConfig(SURT.fromURI(toString())); +// } +// +// +// public StateProvider getStateProvider() { +// return provider; +// } + + +// public T get(Object module, Key key) { +// if (provider == null) { +// throw new AssertionError("ToeThread never set up CrawlURI's sheet."); +// } +// return provider.get(module, key); +// } + + + // + // Reporter implementation + // + + public String singleLineReport() { + return ArchiveUtils.singleLineReport(this); + } + + public void singleLineReportTo(PrintWriter w) { + String className = this.getClass().getName(); + className = className.substring(className.lastIndexOf(".")+1); + w.print(className); + w.print(" "); + w.print(getUURI().toString()); + w.print(" "); + w.print(pathFromSeed); + w.print(" "); + w.print(flattenVia()); + } + + /* (non-Javadoc) + * @see org.archive.util.Reporter#singleLineLegend() + */ + public String singleLineLegend() { + return "className uri hopsPath viaUri"; + } + + /* (non-Javadoc) + * @see org.archive.util.Reporter#getReports() + */ + public String[] getReports() { + // none but default: empty options + return new String[] {}; + } + + /* (non-Javadoc) + * @see org.archive.util.Reporter#reportTo(java.lang.String, java.io.Writer) + */ + public void reportTo(String name, PrintWriter writer) { + singleLineReportTo(writer); + writer.print("\n"); + } + + /* (non-Javadoc) + * @see org.archive.util.Reporter#reportTo(java.io.Writer) + */ + public void reportTo(PrintWriter writer) throws IOException { + reportTo(null,writer); + } + + + /** + * Method returns string version of this URI's referral URI. + * @return String version of referral URI + */ + public String flattenVia() { + return (via == null)? "": via.toString(); + } + + + public String getSourceTag() { + return (String)getData().get(A_SOURCE_TAG); + } + + + public void setSourceTag(String sourceTag) { + getData().put(A_SOURCE_TAG, sourceTag); + makeHeritable(A_SOURCE_TAG); + } + + + /** Make the given key 'heritable', meaning its value will be + * added to descendant CrawlURIs. Only keys with immutable + * values should be made heritable -- the value instance may + * be shared until the data map is serialized/deserialized. + * + * @param key to make heritable + */ + public void makeHeritable(String key) { + @SuppressWarnings("unchecked") + List heritableKeys = (List)data.get(A_HERITABLE_KEYS); + if (heritableKeys == null) { + heritableKeys = new ArrayList(); + data.put(A_HERITABLE_KEYS, heritableKeys); + } + heritableKeys.add(key); + } + + /** Make the given key non-'heritable', meaning its value will + * not be added to descendant CrawlURIs. Only meaningful if + * key was previously made heritable. + * + * @param key to make non-heritable + */ + public void makeNonHeritable(String key) { + @SuppressWarnings("unchecked") + List heritableKeys = (List)data.get(A_HERITABLE_KEYS); + if(heritableKeys == null) { + return; + } + heritableKeys.remove(key); + if(heritableKeys.size()==1) { + // only remaining heritable key is itself; disable completely + data.remove(A_HERITABLE_KEYS); + } + } + + + /** + * Get the token (usually the hostname + port) which indicates + * what "class" this CrawlURI should be grouped with, + * for the purposes of ensuring only one item of the + * class is processed at once, all items of the class + * are held for a politeness period, etc. + * + * @return Token (usually the hostname) which indicates + * what "class" this CrawlURI should be grouped with. + */ + public String getClassKey() { + return classKey; + } + + public void setClassKey(String key) { + classKey = key; + } + + + /** + * If this method returns true, this URI should be fetched even though + * it already has been crawled. This also implies + * that this URI will be scheduled for crawl before any other waiting + * URIs for the same host. + * + * This value is used to refetch any expired robots.txt or dns-lookups. + * + * @return true if crawling of this URI should be forced + */ + public boolean forceFetch() { + return forceRevisit; + } + + /** + * Method to signal that this URI should be fetched even though + * it already has been crawled. Setting this to true also implies + * that this URI will be scheduled for crawl before any other waiting + * URIs for the same host. + * + * This value is used to refetch any expired robots.txt or dns-lookups. + * + * @param b set to true to enforce the crawling of this URI + */ + public void setForceFetch(boolean b) { + forceRevisit = b; + } + + + /** + * Tally up the number of transitive (non-simple-link) hops at + * the end of this CrawlURI's pathFromSeed. + * + * In some cases, URIs with greater than zero but less than some + * threshold such hops are treated specially. + * + *

TODO: consider moving link-count in here as well, caching + * calculation, and refactoring CrawlScope.exceedsMaxHops() to use this. + * + * @return Transhop count. + */ + public int getTransHops() { + String path = getPathFromSeed(); + int transCount = 0; + for(int i=path.length()-1;i>=0;i--) { + if(path.charAt(i)==Hop.NAVLINK.getHopChar()) { + break; + } + transCount++; + } + return transCount; + } + + + /** + * Inherit (copy) the relevant keys-values from the ancestor. + * + * @param ancestor + */ + protected void inheritFrom(CrawlURI ancestor) { + Map adata = ancestor.getData(); + @SuppressWarnings("unchecked") + List heritableKeys = (List)adata.get(A_HERITABLE_KEYS); + Map thisData = getData(); + if (heritableKeys != null) { + for (String key: heritableKeys) { + thisData.put(key, adata.get(key)); + } + } + } + + /** + * Utility method for creation of CandidateURIs found extracting + * links from this CrawlURI. + * @param baseUURI BaseUURI for link. + * @param link Link to wrap CandidateURI in. + * @return New candidateURI wrapper around link. + * @throws URIException + */ + public CrawlURI createCrawlURI(UURI baseUURI, Link link) + throws URIException { + UURI u = (link.getDestination() instanceof UURI)? + (UURI)link.getDestination(): + UURIFactory.getInstance(baseUURI, + link.getDestination().toString()); + CrawlURI newCaURI = new CrawlURI(u, + getPathFromSeed() + link.getHopType().getHopChar(), + getUURI(), link.getContext()); + newCaURI.inheritFrom(this); +// newCaURI.setStateProvider(manager); + return newCaURI; + } + + + /** + * Utility method for creation of CandidateURIs found extracting + * links from this CrawlURI. + * @param baseUURI BaseUURI for link. + * @param link Link to wrap CandidateURI in. + * @param scheduling How new CandidateURI should be scheduled. + * @param seed True if this CandidateURI is a seed. + * @return New candidateURI wrapper around link. + * @throws URIException + */ + public CrawlURI createCrawlURI(UURI baseUURI, Link link, + int scheduling, boolean seed) + throws URIException { + final CrawlURI caURI = createCrawlURI(baseUURI, link); + caURI.setSchedulingDirective(scheduling); + caURI.setSeed(seed); + return caURI; + } + + + /** + * @return The UURI this CandidateURI wraps as a string + */ + public String toString() { + return getUURI().toString(); + } + + + public void incrementDiscardedOutLinks() { + discardedOutlinks++; + } + + /** + * @return the precedence + */ + public int getPrecedence() { + return precedence; + } + + /** + * @param precedence the precedence to set + */ + public void setPrecedence(int precedence) { + this.precedence = precedence; + } + + + // + // OverridesSource implementation + // + protected LinkedList overlayNames = null; + transient protected OverlayMapsSource overlayMapsSource; + public boolean haveOverlayNamesBeenSet() { + return overlayNames != null; + } + + @SuppressWarnings("unchecked") + public LinkedList getOverlayNames() { + if(overlayNames == null) { + overlayNames = new LinkedList(); + } + return overlayNames; + } + + public Map getOverlayMap(String name) { + return overlayMapsSource.getOverlayMap(name); + } + + public void setOverlayMapsSource(OverlayMapsSource overrideMapsSource) { + this.overlayMapsSource = overrideMapsSource; + } +} diff --git a/engine/src/main/java/org/archive/crawler/datamodel/SchedulingConstants.java b/engine/src/main/java/org/archive/crawler/datamodel/SchedulingConstants.java new file mode 100644 index 00000000..67909524 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/datamodel/SchedulingConstants.java @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2007 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * SchedulingConstants.java + * + * Created on Mar 23, 2007 + * + * $Id:$ + */ + +package org.archive.crawler.datamodel; + +/** + * @author pjack + * + */ +public class SchedulingConstants { + + + /** Highest scheduling priority. + * Before any others of its class. + */ + public static final int HIGHEST = 0; + + /** High scheduling priority. + * After any {@link #HIGHEST}. + */ + public static final int HIGH = 1; + + /** Medium priority. + * After any {@link #HIGH}. + */ + public static final int MEDIUM = 2; + + /** Normal/low priority. + * Whenever/end of queue. + */ + public static final int NORMAL = 3; + + + private SchedulingConstants() { + } +} diff --git a/engine/src/main/java/org/archive/crawler/datamodel/UriUniqFilter.java b/engine/src/main/java/org/archive/crawler/datamodel/UriUniqFilter.java new file mode 100644 index 00000000..a93ddf94 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/datamodel/UriUniqFilter.java @@ -0,0 +1,148 @@ +/* UriUniqFilter + * + * Created on Apr 17, 2003 + * + * Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.crawler.datamodel; + +import java.io.File; + +/** + * A UriUniqFilter passes URI objects to a destination + * (receiver) if the passed URI object has not been previously seen. + * + * If already seen, the passed URI object is dropped. + * + *

For efficiency in comparison against a large history of + * seen URIs, URI objects may not be passed immediately, unless + * the addNow() is used or a flush() is forced. + * + * @author gojomo + * @version $Date$, $Revision$ + */ +public interface UriUniqFilter { + /** + * @return Count of already seen URIs. + */ + public long count(); + + /** + * Count of items added, but not yet filtered in or out. + * + * Some implementations may buffer up large numbers of pending + * items to be evaluated in a later large batch/scan/merge with + * disk files. + * + * @return Count of items added not yet evaluated + */ + public long pending(); + + /** + * Receiver of uniq URIs. + * + * Items that have not been seen before are pass through to this object. + * @param receiver Object that will be passed items. Must implement + * HasUriReceiver interface. + */ + public void setDestination(CrawlUriReceiver receiver); + + /** + * Add given uri, if not already present. + * @param key Usually a canonicalized version of value. + * This is the key used doing lookups, forgets and insertions on the + * already included list. + * @param value item to add. + */ + public void add(String key, CrawlURI value); + + /** + * Immediately add uri. + * @param key Usually a canonicalized version of uri. + * This is the key used doing lookups, forgets and insertions on the + * already included list. + * @param value item to add. + */ + public void addNow(String key, CrawlURI value); + + /** + * Add given uri, all the way through to underlying destination, even + * if already present. + * + * (Sometimes a URI must be fetched, or refetched, for example when + * DNS or robots info expires or the operator forces a refetch. A + * normal add() or addNow() would drop the URI without forwarding + * on once it is determmined to already be in the filter.) + * + * @param key Usually a canonicalized version of uri. + * This is the key used doing lookups, forgets and insertions on the + * already included list. + * @param value item to add. + */ + public void addForce(String key, CrawlURI value); + + /** + * Note item as seen, without passing through to receiver. + * @param key Usually a canonicalized version of an URI. + * This is the key used doing lookups, forgets and insertions on the + * already included list. + */ + public void note(String key); + + /** + * Forget item was seen + * @param key Usually a canonicalized version of an URI. + * This is the key used doing lookups, forgets and insertions on the + * already included list. + * @param value item to add. + */ + public void forget(String key, CrawlURI value); + + /** + * Request that any pending items be added/dropped. Implementors + * may ignore the request if a flush would be too expensive/too + * soon. + * + * @return Number added. + */ + public long requestFlush(); + + /** + * Close down any allocated resources. + * Makes sense calling this when checkpointing. + */ + public void close(); + + /** + * Set a File to receive a log for replay profiling. + */ + public void setProfileLog(File logfile); + + /** + * URIs that pass the filter (are new / unique / not already-seen) + * are passed to this object, typically a frontier. + * + */ + public interface CrawlUriReceiver { + /** + * @param item CrawlURI that passed uniqueness testing + */ + public void receive(CrawlURI item); + } +} \ No newline at end of file diff --git a/engine/src/main/java/org/archive/crawler/deciderules/ClassKeyMatchesRegExpDecideRule.java b/engine/src/main/java/org/archive/crawler/deciderules/ClassKeyMatchesRegExpDecideRule.java new file mode 100644 index 00000000..ee2a4880 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/deciderules/ClassKeyMatchesRegExpDecideRule.java @@ -0,0 +1,61 @@ +/* ClassKeyMatchesRegExpDecideRule +* +* $Id$ +* +* Created on Apr 4, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.crawler.deciderules; + + +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.crawler.framework.CrawlController; +import org.archive.modules.ProcessorURI; +import org.archive.modules.deciderules.MatchesRegExpDecideRule; + + +/** + * Rule applies configured decision to any CrawlURI class key -- i.e. + * {@link CrawlURI#getClassKey()} -- matches matches supplied regexp. + * + * @author gojomo + */ +public class ClassKeyMatchesRegExpDecideRule extends MatchesRegExpDecideRule { + + private static final long serialVersionUID = 3L; + + + final private CrawlController controller; + + /** + * Usual constructor. + */ + public ClassKeyMatchesRegExpDecideRule(CrawlController controller) { + this.controller = controller; + } + + + @Override + protected String getString(ProcessorURI uri) { + CrawlURI curi = (CrawlURI)uri; + return controller.getFrontier().getClassKey(curi); + } + +} \ No newline at end of file diff --git a/engine/src/main/java/org/archive/crawler/deciderules/package.html b/engine/src/main/java/org/archive/crawler/deciderules/package.html new file mode 100644 index 00000000..35e86763 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/deciderules/package.html @@ -0,0 +1,74 @@ + + + + + + +Provides classes for a simple decision rules framework. + +

Each 'step' in a decision rule set which can +affect an objects ultimate fate is called a DecideRule. +Each DecideRule renders a decision (possibly neutral) on the +passed objects fate.

+ +

Possible decisions are: +

    +
  • ACCEPT means the object is ruled-in for further processing
    • +
  • REJECT means the object is ruled-out for further processing
    • +
  • PASS means this particular DecideRule has no opinion
    • +
+

+ +

As previously outlined, each DecideRule is applied in turn; +the last one to express a non-PASS preference wins.

+ +

For example, if the rules are: +

  • AcceptDecideRule -- ACCEPTs all (establishing a default)
    • +
  • TooManyHopsDecideRule(max-hops=3) -- REJECTS all with + hopsPath.length()>3, PASSes otherwise
    • +
  • PrerequisiteAcceptDecideRule -- ACCEPTs any with 'P' as + last hop, PASSes otherwise (this allows 'LLL's which + need a 'LLLP' prerequisite a chance to complete)
    • +
+Then, you have a crawl that will go 3 hops (of any type) +from the seeds, with a special affordance to get prerequisites + of 3-hop items (which may be 4 "hops" out)

+ +

To allow this style of decision processing to be plugged into the + existing Filter and Scope slots: +

  • There's a DecidingFilter which takes an (ordered) map of + DecideRules
    • +
  • There's a DecidingScope which takes the same
    • +
+

+ +

See NewScopingModel +for background.

+ + \ No newline at end of file diff --git a/engine/src/main/java/org/archive/crawler/doc-files/processing_steps.dia b/engine/src/main/java/org/archive/crawler/doc-files/processing_steps.dia new file mode 100644 index 0000000000000000000000000000000000000000..5a563aac2ea9d1f407e38472678e82c00ccb4100 GIT binary patch literal 1405 zcmV-@1%mn?iwFP!000001MOQ~kDE9YeV<=}nAgU^d<8nqRJ+nvyHcyIw5|3ra=;

KzfqKP2R`B`xtHBVoHMfAaja0Z!E<$qydMFJcGOo>tIW=e!iXskD4eUetHX~v z&D>0LelGalq5v-$MnCFZYvRXsySUzBR&=}BMllg4e7`N3)c8sMKeNfKVqrvs``gc$ zbIfkjVP!W@{;CFB3&b~w1%#Hd#QcQmWaz*$k2BF8U^z5EI5Yqo8i2%$ZNxZ3gqJ~< zQ50fCQyk8AxI0elp-`mCr9fbJ?<>N2^uqX8D2$r~klub~PFofRjQE#Hn6s$y@<@EX zUOWmkU8(o|K;9onOqL<88a<)BH~0H~+!t5ppv0%gC9@&*Rx9p@! zW_g<$PwUKV`qHcy?V3(ZdaF=)4-g^RW0tx9{Zw+5Y=JYyIh%!O`5*Rp7T)(SpOuO7 zo5#rm;d}MeL5Ti=_;dAdEk8ZQfP^!le5Lu>MyDhK(3n~844@m1;}#qZ^Qa#;jn zA`C=>5LW@+K?+#OwFR{9z&OO4g%>eeuJ=8MctAQRN;%AO)i@MML#w^p_+@^BoByNb6)gx2&@7lf;>3PzMy3L&|# ztk8yp`?-o3L7}R|e4kKEkR^TgPaHmAPCRt31nq}VTB?fW($1Hw3rL=}Re5azC&_CQ zh_pV>>n><{3BS;@YnmL8XpjQB!PRSs>26a|+b&hNDuJ#mGSid3b{1SHFSs+PTkmE- zb4}A6RB+FM?p)y7;?e<5cig6WD<5GKy>)dN`*Lfw~fU1k0;KM#|1@4!M&E?LLFGnd0vl#JD}hi zvVtoTBk3}s1>mr+6Y@Zb;5tdu32opG{5qkJ5St0E`^&et|1#lo`WN-_)2O-E(_9-E z#`s2|PtE;@BQfA_m~ARl*OXOVORSezw~5$!&uqiS6KD!3y%lJ8c6tvEDZIT!*+hNe zkkZ@DH;ET3LWK$y9A7#B9IjQc!gITvYBbbb<3t^0_Z<%tS3+qP%TV3PF_Ngxt7E;S^_-zaND{j7BJ=*Y@b4T(e2Zv7AgTuDuD+O6kQ7(L?#Ue2x8a}wf`W9<9JG>xbFBhc46@!*=7{S}bI=4jgm6F(>Jl6@ zUWJ3OFAmC_;I`tR05VS7Kj;AWYTLh3koA@_FOT0e2WMiv;E>xtLxO|Wt8h^7iGxyS zBW*aC0G;)sx(@8?5U>4B67IY? literal 0 HcmV?d00001 diff --git a/engine/src/main/java/org/archive/crawler/doc-files/processing_steps.png b/engine/src/main/java/org/archive/crawler/doc-files/processing_steps.png new file mode 100644 index 0000000000000000000000000000000000000000..64b3db75b4722ac43648b855d9ad6eedf9ad959b GIT binary patch literal 15555 zcmch8byQXVw(lZDLRg9nvNF zCZ2oZckX@h$GvZjA;N~e_KNwP^HVEOMM)a#Iu?Q;Sh6xws_=gcf}jjxqQZ9>T94@A zKQu=PS#?ZI%<&(}Gw|b;r!v}(2y*%R#Xpp);KdXKp+;n-#MNDse!X*3R~xQx^`W`a z!779LTnx!m&r~lc_5V(o3eNK8yD7#zK&^D<~^YUpBH%^4K0iudo|Q~8`cpvJ=vVUb2*Gn3KP z9nX}F{E(N&)=oi06croGNDzpj8;$nE)EZ^q|9-~ros@H%v=k(@zW%JBa_YLeEY4+} z70&wO^&0tD_PsnAB_*Y2&-SU8?WMk}hHjl6OzS?dS{*8QbIV?DQ)dOAAdrrOLnTX9 zTQjD0mXp05|BJ>7{_scr(r?O$ik@CqK56SEwmnRLiAT<(5fKC!(fmA?%r76ozMejN zR#9yuC57%MM{&JY$OqjZnH#l2HFRd@x8p)rar(^R)@-k`0Q_ZmT%6^-+&6EqTcY{d zN4A7t-OmUN4c!<<+a)pd+*%pT^J8rPAtq1uqA=t#<}=S%hlhuHpPtfiaD2D<1{cK8 z`}d_&Q$#1|#p~Cv4Sh}r*YIk`EA3{Dm`#diGte$$>H1+wM+x9Pe*72-i-|F}rolm; zd15xEX5qaOm#4w@i;*D3VDsm|*1>v2T1NDh<8LSM-}MJSMs$2n`$k5zbamw`u3_CX zHO-Vr`{cI6Ad4R;g@S@op=QxA!$ZzcA-IBqS$xeHEOkZm7Sga z`LkHaeMu34yLW|MH$P8LPgi_cBeTisn%US8eKN)~+4{IYJ1u2NTt41n>t}P+I41qb zR(wuvt&osVl1*%MbO;%DONILy1-ymZyo7ZsGZf4IhgwN#=d)uZ%hg1_6$Nr zrG;uBul-UZ?;Y=xu>##nE!LQV@)1F_@~Hb6xdjDYhns17>=ts7UaAisaJuhW>gJK^ z!B$&aD@7X~*^gA;zD0b6nVo&A)S{>Caer-HoopDCOL)AOgiX-8kh_P6aw6}Oo@C+X z=H}CKX;B^Ka-Gw?VZ)8dn#z(A)z~)U;ukMon0Ljckt2cp?6l!a)8}U{US4(ex02mg za@lk%@^#9Cy3KWonhB;K^CVDtABrE?n9sF^=V=rOcpdTNlDlocFDw-GI@Q?;zoBR@ARpLlydC7U zyVOfhPp_~lMezn!oI}s*#>Q>pU_~?T$GHKxR1uYU+!o)~*48K}!e9IKj*i}STu?w@ zWw;)0(wI?N$>$EP_+KSK6*D!ZJG#=FQ(DT+#YKP+v+KRr$P3|C(X~21J4I$zSC3Z< z8*oY_nzjz%s(E>Np%}q0TBcRCK4fJ0O$a25`gm{7)8MrK-dSv)&3yY7$FGnnxn^U$ zvar0|d4Fv*b3+o(1a`~svA;GnG~{)>Pv&WMxC=LGvZqHmOLel=&2eQQSCWaqp2Pe1 zeC%?6jw654!h-o^-dzsL35Ttp_p(&)T709}C6wqH$W>eHiqBAIxouGUKE>9Oxlj71 z7IzcDYvtD#lxaSX28V|imzS>~&~GK52D+Ra+KZ&%YJ|M{ZZU(OEc9w2PF+(I%{M2< z!q9L!)bDDQ`2o4 zw`lyfHm2(G51?N_^=+~13hEo^uca(Ny*0D<^BTpGxXH*M4EDV zlP%3R8XJp=H^eIas)J@AmuuBPsVvgQZ`mr z65`^RBHOWpO0`%EG*;*4q+U&_E-)GvYq3sl>Sd``J?>|q3|;*7>vXG?YBk+!f7F78 zhUVM1Z-k6W1F~H2ZQ&@ry}fO1ZSCrRXWK4dqm9$3N^ zw&y#;!^7dNumfnamlV@&8?;!7=h@rlx1(=qR##R1K0fx@?Ujydj5>B!fEtZu)&BY8 z2Pe!Yxa0D41XrWuUVQTB57>;$`lM$IUCs72wlEtT+w$`Ag9i_)qfSmvUc5k!mb--1 zxNOurE-HzOi#s~*6g_IJuo*R0(Q7Ea6Fp6hi>ysll|mO!O-=pq;lsy|N+fKIjEgNH zxzpX z#46GeT)TEH{8x51gQ%!zhl=?QITI0cS`2lexDQ`tMTUBzBvZj+zAIW}h#%&do`7$3lXFgWLU^ z#axenRcaO*=xb}|s^-FL-BY9q9iqlfuVK~dS3tqm?|c)4qky7eW@bK(D0|e1;eZt$ z5mDm#+o40~%9Sf{gH>~*Oyy-{{#^K%FJD6Q^p{BMwH&azqnNH810COfwk0NpI9FZH zZPG+8k$KLc3TAO|usjr4!X0bYs7F{Ovzs-J-i@D~W?Mqge3OzW)IRdCOT>TF$aC44 zVj~Frq_+;RMmbqX*04$aYIoC^Cx}({{5!(ThgmVb4Dv|?d>SOa8|+^P1OBE1-d9k zjjynHo>Sl-NzUR$#l*aN^=eF~*~Dg~2nDIIo&1nudHB|@H~k*+^k>r*e+kad8u*W1 z0&=Y^FTZo=&XcLy(T@yfDojL-N=d%c!WoUBoLV|*s&%+Bk^09?L1cVM-W;plMMcC#DSV>4i^(O`=6?H)iU!Z3Vb_^2!4 z&Q-*~=XCGO@GE}0x#_O}lex}<##G^LrsBcMBEL<;%4y!VGNyQED{8cVcvDCOylA8tZI0^-|jEPQ^l18=()CbZ$-#WA;r=SuSO=N^HLT|IvyF2R*G=EJv7VmLlbQwDESN)SY2r*o7?&=M>U2_X zua)-j^6^F5GIDS{{k=UulwUbssIh`8ucLDv$;`?c)p?EPo5xN4KY^1ecC^plJ3YJD zBAygz+`_{w(|IB(i6%><)- z_csj=&d$+tSsy=slq~Ex`(n);`Kh|PrMdZm_caod0B^2KIzc#U`sB~tc#K-5rLC>= z8X6kDY%@~e^m4 zE4|9lJ~(SmcBn3}Ff~CSgF;;DG5@2l=(~i%mdra#2Tmia@t4@lC{Hbxo|O+pdA%4` zCB`Qv)&S&o{kqP>hi2An`#2S<#NtPSYkRD~-0(9S8Vz~~ie^hO%XMy$k!h=|H<_0{ zDQKP|qm_rnhUNLtW!wRn7y#~3oo3&er~FU{eg>UJ=z~_}F6$F_ea<|g@}695zczm~ zGD`6mMv$Jz@ z@&?JJVj7)-ikh0^AB_P=f$Z$;P`C@p!X5zVpMZIU;rM-Zdg3Y$0l{CL$st_zA@~F>yRsEeC`2 zNovdypt1IAs<7GFZ6Fym73r0ge9X*O5G~fB#S|ZrFcq|tfzeWncS#r9{>HRu82{^c zMLy@=4iW(Vt)1~!*_=0~gdGyY!owdq<26mSwY4D#E^esD!MZld7dd(6=#HWy4n(Wo zv&tDCuf`0P0D?zMY(9y)tf8~=Jxvnfc($gc7jxP!nX4YrT#S1|!1SKIH7lUM=EWTp zBWhuG;~pqcV&u@%8-RyTrw%v44|pFzpy@(ad@wOv$-**=@d2w&nI-&85ctwb`m_=2 zs`UN)sGsLMV+)vq`LRGG!0?45J0o5@5UgKWS%E{kefxHYa1#%q&wbrv_FDQIJP zA3jJ(O0wM^*?7Bg@_Ps2u$!u-*&(dA;Wfv6dnLB>9iJWLiFXTI&U8$*{VWF3l_HwL zfV*nH;3mW*f+LBNtt$ ztxd+lVlgZ%?4koRJg78;uINBA=kU8D`qt#c#L4NY=+V;6j*3jf*39Z^_x1>SptW#; zQbc{~rD%&)uF%i1MsnNUJ*|imqn>f7IxQbqpRC!Ks8Ul=`93ir1S1ay{^t$R8xDzL zCBOa+gK7%>I|j|C<>RX?Dk`e3)?p=XIGFON-u)ZEB=xC=pD+k zvH(RvZ=t27y>n+!f<8*i<>T=w-g5<&(V?OHk*%Mw!euhagZl>t<`x%oI&%$S5XVGE zlSx}YeoQmY75}F3hsu!Fg9qXjogE#hXlTyMAt52vHXt#~&(E74pPe4Vw>*0ZeH0WF zN%8U8y1IEO@m%=3-W*Xic%-CS+S;-eu$-Bc0|r~A7$Ub^g|0@c^bHMRAj!$eJZ^86 zE3>{fHln0{`V`0hzBld1Q+^v>fSiJN@1h{V&FNk3=A|Lio1}ATLhDGXYHnMFYVN=d z=GnujKVarAz+@4Tso`NhCMKp^w}KfSoJ*;$zj#jBp}#uJS>?84?&QQnR)60uiS2Hj z06;txYLFBdLOD+>2J*E_kr~i|oWw(3!qft?m)QMyg+!K0&w4-(a7srPoLN)|nOR*n z&+4cD3(|bFK+KXh2YW*}tcO?=gj#lwb%L-l+1c2BY35c=dwcAy3`R#svqXRR^ogI7 zGdwetlrud)Um^C%d`I*~?T(V9q@=X8w49t=!-tWgM`%^AWMpM&Hu^$jWn^R&6=53D z=}JhT0?XoIq@#<9iUM8`az8^O?=eAliOpz92svLxS(!|hYR4+2+L!_jEZTSP^!4-r zLk;C?o1|f@g_Y)$quMf`a;0kIo$im@p-}Vk#xR81Yr#_VIIsZTcC}ltf`hN7^h}pp;Kp{|nJV|b5$U-2E#&&0 zpvE6BU%jHd>(Hd-3zR_G#)cK~1Z;j$9tT(BhlVuZHuEO3Ma%uNV0UxqHL|{j_7f`a zDluzO%3~n`K|wd6WSs(FsO!U05vS{ipqtz^X~71B934G4UGkc9b-2*Z+S=O6YOL0+ zRKL3_EIK+%g$Z<;-i<)nJba5Rg^!Ge))?(Qli3y~e2^UDS7 z#aKOlOm@eL2%%$O(7AzL%-1$s2{?RtYJ4{qMSqYOIAX(g-O)-OSG)~%8eZbf$aVv- zy@$x=<|b*|941=b1LzI()*^~=9A9R#n16~53=KsrPwMOIJ32aECJ1`&vC-Gp$7#~? zdBd^8$Z=k)&6I1BRmr5~4Ko|t4I?pg%0(w<=eqqd>pAh**x2DOewX7D@-fMTU~uD7 z3cXJxSSuekFq!*88o9ywQNO0bd3EUI#1kI6~>Lx}<-!r#G^<`GeHZ?WvN)}FxPrz80LwidpTtg60X_j(!|&Nq>$qh%^Fj+jO#S zpuMzbR=!5TX;>JU36G5>AtW?e8HYi)w6v5qFv|om72we&)aU9fcg})ZXW^12b;l5Z zaC;@O`)z1w&IyZ%2<53MRvFQa>><6fymHcCSZjw%VV(0rl*$Z|%Z#AG$Y!aMkdWx+ zmN*tgr#Ch?)3dXGqvm91$N6{VwxWU;7GAX6JrRdL8X6T9)%b@Ji=t>QztGUov;N$; zIMTmBRY8%Rfx$ulmoF{Lb%=u>s;LoDP}o@Cdxlk^x^M*thvAl$QXB`$sVQ+;)V;7X z91mf6F?nrtoeqahoeouF6O%%{Dr4(wIG=r4K~sR%1-DsQS=qvZ(XV7J3`9IQnhV8N zNLU!I+a*+S8JX96d7}F+C=IZ(z{0QxEO|uo0i1wG<|%lI={}TxetrNW4`xF}_d&Pr zq&cP}EA~?axEY`AgpmaIPmYOC_OrhiQy4K^fB2JRsxn`Qw$;#x zEiIW4S65e0DE6*pP-B%vPX`(s(GWmrAjHW>Mt)NTsfTfL5!6d^a`JfMKz>pf$w2uo zA)q-PA8aIcnD?o|Qez@Q6@%7IFU+(pEtaMdwsk0xrou|Bwvbk#4!9SDMHFfQf$D~X zu(n7=Lqn?Hm80;Qhlj@>!M6474FotzLSo`&ZJM91JbZk?QnU#0)PSu@qMj^xJp~kP z=#JouC@1i=CoGb;MSgoH=)$0!ytck>psoFQQ}y}J@+}ajMn(zM2tXWkYvQN*SZjJorH$A+UK{!Z%V_RC*v`D z-T1{At67HMA)W_d8qAc*X)`M8to7-J+p(QDX=yD_nz&vtFsk*DVVh}=^muFMG)+&7 z($Y$>$4)o+7{(sPWi=UdB4GgwJfuPaMG{yNMJX8;hqv=}sa zzq2u{b9esn)xT-)sGBsnxudHK8;Za2Qr>^oJ0lcx)osVh0$XR@JUo0(wpurjab-j= zZVzAypgyp{{O%tgm)G_eVI{WyS1r~Y?wJ|BT6oAq6knZ!sfs7$DLpRE&WZ{O1Kln5 z{~7L{&v^c;w#y2d@v=jeCjbzboInGqyms-;bR}}J5GJ$)fi?3BDYiK?3;dC6<&pZqKE*FsI#2!S2o<>f9m^bDwV9Wr{^{k6SWTp^o74d&gi*OkWN9n1)=ug z!-v3JPtMT#aQD`0Hh6h?HPqF?T?7r)X1e}e2^Jc3_p4X0!gXpk-tyR}lh$rhH#K>P z6alRQ%nM8o$`F^F*QPUujn3=80K#|g-i7rA%~GqxRMx?POLn5zL>ekuQBe_wyR9wz z{Fj^obC%THMq7}U%w`1;Zd!u#IO@B|}UbcGkkEbIS zfcu$)L*KMNkdMPdS8y{xH_y*!O^`Yzl&xyOaA$J)9&t1K*m9*P@u4x+wsaXBgfPgp zdms~GBk)7kl}8`m#&cUx3cK%uY5-fTi{_r4g(4g9>eR?{Dp?5718xymrIy(IXK2G-uGuMk1pyb}_Qs>sW z>K(?#FbqLL&Bcpy)~j>h14IC)#S(3Ga<~QkL#N>nfBu2<*k5OQCH^$<~dz2^+2{_B50`e1hU{F)Vl4QK~cv~`XH|09f6fD8xDL< zg_!eNyy_s}9Jscc4<4kcvjhhP5fBhK*#>zd2!-xN7C`2x#H&o)x(h#jxrH^8x)#i`C%M(;py9K4DwL zhtWRg;|is5wEU_tgi#?5kC3o`XsE(&T4YuPhs+)5BFrX1aBtx>aYYV44>?d4+fCP3 zR8#;S;Zlph35pFqKRal!_?9Gl*9cb}vpTIo6ejN*GVa}mbkaef25*{!NkBx8Y(p0^64ZPR5%X8#Ah_#}0KPNeVPS>H$k- zC0^}{=k|M#W6$#~>29NOF+asOa15BZzfWE^GdD-|jf_-tcNh5f{<$Sj0!a4T`^?J1!{Y?$i8(gPZBsQZtxwj&>|D;c2)GeI z6yw+iL&Y~1ds5Dir_T-4)uluL0yej{Mrv=ik$bFe$l)JfsKy}XcD-;pJ8_%j9|Xw= zP*9@IW?z{1s76MwUajjE#5}mPq%Udy_Tu8;4;2v+v8&q~f~$D!>}Wo3=m=L3&c3Mu ztOfD=J3_fcBqXs>QBqP;K#0B#>0pF|bPu!XiZ>%K?@)rzx81zxqsEAs7z_jg6cxlj zfr(@PcV#0k;uj|T^Frwva9JxVVr*^^1%dM8FJZs4pcp|*5QvJ3YLoNLn=Gi=nj-3! z=EtVy=J+FC@Ip?~$T<-^()9ievJW4j{-8&LiEwSH& ze{e{)n0&8z3}unf4HN*`%PT8Cw_K~uiZQ_tiImGSjS*`iwaHonRu;!?@mBn2(p|^z z|E2^}cOF>rT;J|(Z@)s~oqbuCnfvhg7%~Qm2liuSkCzzTzpE~2*Sd1f^GaNK=DEOp z`!;ZE#&I(qn2XC_O(UMLskYu36=4NKg$v4no zwTFsPgUxADIx1PL{Q8$l-XK@J0xL0iKg$Flug0+JyU%=no|;O%VjOj!Ft{0J-Mm8; zT@#!W_*0;+1+40ya`syC9IQ`vwzb7^nqX*W2X)RvdjwB|+#o{7`xOLM zFZ_D1!M#frOdC{$IJn&F_*uD*a2YNBWyt~!=r8Xkil2;?$;O_wse*|@k6@GYvNAHZ z5@X>9&@nPvfNFzmf=Ln)HO(3w8!O$i2(HsGuUY^<1H^~!ApC-Yi}Ul31G9iuz??;l z!0fAt3E-zA=d(peL`55tlPSMjl|v$g2DwlIQt+vVz()i$jg4HqD9;l8_WGRWU?z5F=uuF5ka!C!3ltK=Jq)|fk8Js^a1ZO105Y5>>0E*z#pIi>t_aW$;^b8Ew=Z5!v-YfyOhfv4Jw5l?T2E}Lb zuLAT9Zn+;mbPVjgT3N9I+6R1a(CUJ=!DXZZ=9V|e5-lf^VyLEuhL|p4Cw4Q?VrPc& z0}udnLx{2mSS<}rT1}0vW^)v?TH7sZeRJe+w_o`lEj~zN9FM+2gOha!$pEZUfSV~P z)Ae4*OMX4?1P{i_t?3$}rgp&%BVW6YfJU+T+Y@Hi!#s_>{r#1`y#pEcSVl%hup(jm zXi%ra9vf+c<$oct4iCEk00BIhGu;hQI9Lg9JLiL>(g7$X0k#oeT3*&JHum2V{~n;4 znClDB1g8Eo7Z)mCn-PE?z^d5PC1f8yyasJx6^D=>;!dAjw}A8OgETU3*KoR$S7=o} zwzJsNlXN$7AT}(FI$AEodFk?RZ~zV5m$P8`fsL7&nF*3i_%7rU@NeD+tVS3C z)YQPfuuUV7^ClOcg78D+a>My?LoAza_MjCSDk``ckh0Lo^MXYdN^w_W1u`@PX1r$1 z;WFwlFnw80eoItg$fhvty|o$AIr%;efkvqh9^%7yv;kL+jyAX=FcaT1&u01Kun^*v{H*b}EDz5kTT>I*-gE9`}Ucm<4qD4fo}6?6$fyz z5Z|>mn*;^G&1WYU2}&CMMnJ>>PA;aLVT!oB3g`w)D&X21H(_>sOXSK{Val}0V)C;t zANvd5=QTAmZJ-U+4&+cZR|l6oi_Sa|_B~cx|*4#RwW3z%9=DVQwQ>R&)7;Q{KnF zz~0f+(D<{RRErx)`AFvIo^^)J>++0~j1p_^Ofqu<15uGS70~k zDoi}rLv#wVWqCRQ;q%%~jP&)V1<{}Aol^^2J4bqe5g3|!?3&i-7qtuhkI!m=2^|0H z*WKsZ@J&#%u~Z2N3DKaZ08Ko@@(Lprc6SmN|MGoUK2?+lM;zdrrRDpAheGLDS^nVL zK&C~$Km#a+r!QObHDJfeaU749R-K)O3Js?ru{SVK2Chs^LiP_hCad!6N+}{(2#60y zp;YIXzKs>CDW2O`K*w-#QC3i()4aJa2BMbT4jP~&E_)+)_v(U&G&l-?tPzDo-jJ+8 zt3tyD5s!VFzj?1e-ZM90Sy@}Bz?=eS8o|9HRrPuV;I~_F5dD8&?zQN@r3&HTB-%JvdLIY7sgJr)Lr`?TRBN_0OtWauJ*U?7`2 zJNK_dolHU*l+|jMDsd8Q=HZY>O9hv@+Ap`)@Y)_O4-~fBEu-lasSYXy?su zF0>W>a}LOMpor<{Boz4bBcBA&{=0I|n}+NR%aA`^TcaC?Gz&GJW%(Fj*9eJ)=F)cp z$IX2W+$Sh^J@#3i>Wc1h8ARJMBoxgW7=gswx@IitLMaC#bYv&)PienW;Cgv7D`$T~w33OXsT41BpS z+6#GVKYOZxI!a*(EvJEIPDdvtnw6c6!PuUK0)YOt6z%EnGvKkl-szVw!W_vUe3wEV zB;y==@+0v`k@5vP2iq3nkl+G>8E*r5jV~G5QJ;Adr)lIt zln~^H5u1vOWV7UxTnG#L9gx5M3UTMQ2qT=!0gb<(XqTqt+NxwglLpx=Nz_LKJY?X< z6&248H-AJm-V6KE2qU_(Qu89xkq&wpq~AfA#hNPn?l&ms?CcDgj$e6&^-xkr%a8{F znU}wNFDwl@T1`!R---Z~-G%oq?6D78HMU%IS06~cz&jycKmhsfzl1urR`BEj_k*wn zb|ApBe&-jk%bI7uj6KV>Ua>t?S0|#NP#|h;ZU$F`lbq_&%ZpUU(pNyJAe~|OK?E5j zCWu6#hygUgiq(*2%>=^0!h(f>7^L)0pjln3g>?_a^vGKPd@`q|46?t{`pNx4cLYe0 zR{jP03Xkj3T7jVgk0!td2+e|ynkxzfzLcbTNPFW5T}O+lcG;k;vIqK0>dMo^0KgAH zw7I-|(=e=K(+_N{aBda86OIL2+M72E-Wr|$CC?!@qT$UF2v8;T=d*0|fynnbY2QbORbF;6Y$6x(~t| zv2=^5z6>$bEJh_6-YS|>g}#D_QOfm-9SsBUT1@s}=j7~NS)stpE2JlF6`-F>irBF> zD;N@Ixxhi~8B~VsvGC-{B*=eL&TEw8PCIOXeSBQoX7mu4$`KPL? z?stOF%%d^^oc)Vp9ro)VQDHx7o-OBY(?8O6>fh4H-RjPu)8^-QQD@80^@UJ@QM0|X zlhJ$sHYuW`qtjOLPdr^x5@8d`@WpkQsXQL*eJK25%M}6w7>rh(7~&C z-S;meAo`B_8PX$`a8{?~L}Yk+0LSl*g4{MRp#FFGc_9xkgFuJnFOks;cO!!TZ@FT% zG+=sSA87!hwbeLv=IO!{I)WRu{SdJ2?S(+E1i|#bLEK4wmMq>5sZv%z1)v!~7#0r{T_XgJ;1MP7%5WVM^H)St0wIXIN8 z6R-UX7QD#+(q-r5KvP$Hh3X4AkO;y0%uG6c3cs%GivrZ;3b5CA~1zHVGh1E3kO1JuvJh%g8Op1<`IN=*=Qa{wm$IfI5b z4W1AI=EmXM`apoRNL^mO^ZkR23!Y;?(~FlVM%e%n7`cO~7OS|bS~eCITAz6jh*N@| zc9SzaO`IX}BItq4{P^(&q8)E0Ip-G^BAQbmD{8cP{aUdupxyTTb?8JeAm22ZZZp&Z zXoCp-<}oYxh~6kS$ZdQU{h*sU`AK2mH1;?H8!_e@}TMhNpQ_hmtB`9|iB$Wo#HLzY3%TOhZotEtII91Oj8Y<&E3dCM2y z{Fe39ypJFrfUSas{sOvHTwEN~l>@)u-CrBj+Iv@Svaw+yfR1h0novFulKuBQe5d{L zwCkLJQr{uqQFiL4j&2&p%)cJvs}ox0_dn;K%>GY z$We%Y;hhdU#{i$hLl~rAbB<*3v5yN;6_W71>qY#<`{=P66}}0m!o7AeMS)ybbvlHO zNR(Y^%!oZ-8&QVbCnO>o7#RU>aQ*uAHUY2K%$rMbt?liPFY@CLB?EBMQdj}JzI=q?cwMD6r@HzG67|EtwY*OSVmo09 zOQbyoat;J*kj>>Ssc~nZ#IHE$6Muc;lt-7%=J6tgq{V@v+Ssf?g)Nmr^8WO2HWc-_ zk)`@wEK6odNTo9checkpointDirectory. + */ + private transient File checkpointInProgressDir = null; + + /** + * If the checkpoint in progress has encountered fatal errors. + */ + private transient boolean checkpointErrors = false; + + /** + * checkpointThread is set if a checkpoint is currently running. + */ + private transient Thread checkpointThread = null; + + private transient CrawlController controller; + + /** + * Setup in constructor or on a call to revovery. + */ + private transient Timer timerThread = null; + + public static final DecimalFormat INDEX_FORMAT = new DecimalFormat("00000"); + + /** + * Create a new CheckpointContext with the given store directory + * @param cc CrawlController instance thats hosting this Checkpointer. + * @param checkpointDir Where to store checkpoint. + */ + public Checkpointer(final CrawlController cc, final File checkpointDir) { + this(cc, DEFAULT_PREFIX); + } + + /** + * Create a new CheckpointContext with the given store directory + * + * @param cc CrawlController instance thats hosting this Checkpointer. + * @param prefix Prefix for checkpoint label. + */ + public Checkpointer(final CrawlController cc, final String prefix) { + super(); + initialize(cc, prefix); + + } + + protected void initialize(final CrawlController cc, final String prefix) { + this.controller = cc; + this.checkpointPrefix = prefix; + // Period is in hours. + int period = cc.getCheckpointerPeriod(); + if (period <= 0) { + return; + } + // Convert period from hours to milliseconds. + long periodMs = period * (1000 * 60 * 60); + TimerTask tt = new TimerTask() { + private CrawlController cController = cc; + public void run() { + if (isCheckpointing()) { + LOGGER.info("CheckpointTimerThread skipping checkpoint, " + + "already checkpointing: State: " + + this.cController.getState()); + return; + } + LOGGER.info("TimerThread request checkpoint"); + this.cController.requestCrawlCheckpoint(); + } + }; + this.timerThread = new Timer(true); + this.timerThread.schedule(tt, periodMs, periodMs); + LOGGER.info("Installed Checkpoint TimerThread to checkpoint every " + + period + " hour(s)."); + } + + void cleanup() { + if (this.timerThread != null) { + LOGGER.info("Cleanedup Checkpoint TimerThread."); + this.timerThread.cancel(); + } + } + + /** + * @return Returns the nextCheckpoint index. + */ + public int getNextCheckpoint() { + return this.nextCheckpoint; + } + + /** + * Run a checkpoint of the crawler. + */ + public void checkpoint() { + String name = "Checkpoint-" + getNextCheckpointName(); + this.checkpointThread = new CheckpointingThread(name); + this.checkpointThread.setDaemon(true); + this.checkpointThread.start(); + } + + /** + * Thread to run the checkpointing. + * @author stack + */ + public class CheckpointingThread extends Thread { + public CheckpointingThread(final String name) { + super(name); + } + + public CrawlController getController() { + return Checkpointer.this.controller; + } + + public void run() { + LOGGER.info("Started"); + // If crawler already paused, don't resume crawling after + // finishing checkpointing. + final boolean alreadyPaused = getController().isPaused() || + getController().isPausing(); + try { + getController().requestCrawlPause(); + // Clear any checkpoint errors. + setCheckpointErrors(false); + if (!waitOnPaused()) { + checkpointFailed("Failed wait for complete pause."); + } else { + createCheckpointInProgressDirectory(); + org.archive.checkpointing.Checkpointer.checkpoint( + //TODO:SPRINGY + null, //getController().getSheetManager(), + checkpointInProgressDir); + } + } catch (Exception e) { + checkpointFailed(e); + } finally { + if (!isCheckpointErrors()) { + writeValidity(); + } + Checkpointer.this.nextCheckpoint++; + clearCheckpointInProgressDirectory(); + LOGGER.info("Finished"); + getController().completePause(); + if (!alreadyPaused) { + getController().requestCrawlResume(); + } + } + } + + private synchronized boolean waitOnPaused() { + // If we're paused we can exit but also exit if the crawl has been + // resumed by the operator. + while(!getController().isPaused() && !getController().isStateRunning()) { + try { + wait(1000 * 3); + } catch (InterruptedException e) { + // May be for us. + } + } + return getController().isPaused(); + } + } + + protected File createCheckpointInProgressDirectory() { + this.checkpointInProgressDir = + new File(Checkpointer.this.controller.getCheckpointsDir().getFile(), + getNextCheckpointName()); + this.checkpointInProgressDir.mkdirs(); + return this.checkpointInProgressDir; + } + + protected void clearCheckpointInProgressDirectory() { + this.checkpointInProgressDir = null; + } + + protected CrawlController getController() { + return this.controller; + } + + /** + * @return next checkpoint name (zero-padding string). + */ + public String getNextCheckpointName() { + return formatCheckpointName(this.checkpointPrefix, this.nextCheckpoint); + } + + public static String formatCheckpointName(final String prefix, + final int index) { + return prefix + INDEX_FORMAT.format(index); + } + + protected void writeValidity() { + File valid = new File(this.checkpointInProgressDir, + VALIDITY_STAMP_FILENAME); + try { + FileOutputStream fos = new FileOutputStream(valid); + fos.write(ArchiveUtils.get14DigitDate().getBytes()); + fos.close(); + } catch (IOException e) { + valid.delete(); + } + } + + /** + * @return Checkpoint directory. Name of the directory is the name of this + * current checkpoint. Null if no checkpoint in progress. + */ + public File getCheckpointInProgressDirectory() { + return this.checkpointInProgressDir; + } + + /** + * @return True if a checkpoint is in progress. + */ + public boolean isCheckpointing() { + return this.checkpointThread != null && this.checkpointThread.isAlive(); + } + + /** + * Note that a checkpoint failed + * + * @param e Exception checkpoint failed on. + */ + protected void checkpointFailed(Exception e) { + LOGGER.log(Level.WARNING, " Checkpoint failed", e); + checkpointFailed(); + } + + protected void checkpointFailed(final String message) { + LOGGER.warning(message); + checkpointFailed(); + } + + protected void checkpointFailed() { + this.checkpointErrors = true; + } + + /** + * @return True if current/last checkpoint failed. + */ + public boolean isCheckpointFailed() { + return this.checkpointErrors; + } + + /** + * @return Return whether this context is at a new crawl, never- + * checkpointed state. + */ + public boolean isAtBeginning() { + return nextCheckpoint == 1; + } + + /** + * Call when recovering from a checkpoint. + * Call this after instance has been revivifyied post-serialization to + * amend counters and directories that effect where checkpoints get stored + * from here on out. + * @param cc CrawlController instance. + */ + public void recover(final CrawlController cc) { + // Prepend the checkpoint name with a little 'r' so we tell apart + // checkpoints made from a recovery. Allow for there being + // multiple 'r' prefixes. + initialize(cc, 'r' + this.checkpointPrefix); + } + + protected boolean isCheckpointErrors() { + return this.checkpointErrors; + } + + protected void setCheckpointErrors(boolean checkpointErrors) { + this.checkpointErrors = checkpointErrors; + } +} diff --git a/engine/src/main/java/org/archive/crawler/framework/CrawlController.java b/engine/src/main/java/org/archive/crawler/framework/CrawlController.java new file mode 100644 index 00000000..5b820b3d --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/framework/CrawlController.java @@ -0,0 +1,655 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.framework; + +import java.io.File; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.PrintWriter; +import java.io.Serializable; +import java.io.StringWriter; +import java.util.LinkedList; +import java.util.logging.Logger; + +import org.archive.crawler.event.CrawlStateEvent; +import org.archive.crawler.reporting.AlertThreadGroup; +import org.archive.crawler.reporting.CrawlerLoggerModule; +import org.archive.crawler.reporting.StatisticsTracker; +import org.archive.modules.CrawlMetadata; +import org.archive.modules.ProcessorChain; +import org.archive.modules.net.ServerCache; +import org.archive.spring.ConfigPath; +import org.archive.util.ArchiveUtils; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.context.Lifecycle; +import org.springframework.context.support.AbstractApplicationContext; +import org.xbill.DNS.DClass; +import org.xbill.DNS.Lookup; + +/** + * CrawlController collects all the classes which cooperate to + * perform a crawl and provides a high-level interface to the + * running crawl. + * + * As the "global context" for a crawl, subcomponents will + * often reach each other through the CrawlController. + * + * @contributor gojomo + */ +public class CrawlController +implements Serializable, + Lifecycle, + ApplicationContextAware { + private static final long serialVersionUID = 1L; + + // ApplicationContextAware implementation, for eventing + AbstractApplicationContext appCtx; + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + this.appCtx = (AbstractApplicationContext)applicationContext; + } + + CrawlMetadata metadata; + public CrawlMetadata getMetadata() { + return metadata; + } + @Autowired + public void setMetadata(CrawlMetadata provider) { + this.metadata = provider; + } + + protected ServerCache serverCache; + public ServerCache getServerCache() { + return this.serverCache; + } + @Autowired + public void setServerCache(ServerCache serverCache) { + this.serverCache = serverCache; + } + + /** + * The frontier to use for the crawl. + */ + protected Frontier frontier; + public Frontier getFrontier() { + return this.frontier; + } + @Autowired + public void setFrontier(Frontier frontier) { + this.frontier = frontier; + } + + /** + * Scratch directory for temporary overflow-to-disk + */ + protected ConfigPath scratchDir = + new ConfigPath("scratch subdirectory","scratch"); + public ConfigPath getScratchDir() { + return scratchDir; + } + public void setScratchDir(ConfigPath scratchDir) { + this.scratchDir = scratchDir; + } + + /** + * Checkpoints directory + */ + protected ConfigPath checkpointsDir = + new ConfigPath("checkpoints subdirectory","checkpoints"); + public ConfigPath getCheckpointsDir() { + return checkpointsDir; + } + public void setCheckpointsDir(ConfigPath checkpointsDir) { + this.checkpointsDir = checkpointsDir; + } + + /** + * Statistics tracking modules. Any number of specialized statistics + * trackers that monitor a crawl and write logs, reports and/or provide + * information to the user interface. + */ + protected StatisticsTracker statisticsTracker; + public StatisticsTracker getStatisticsTracker() { + return this.statisticsTracker; + } + @Autowired + public void setStatisticsTracker(StatisticsTracker statisticsTracker) { + this.statisticsTracker = statisticsTracker; + } + + + /** + * Processor chain + */ + protected ProcessorChain processorChain; + public ProcessorChain getProcessorChain() { + return this.processorChain; + } + @Autowired + public void setProcessorChain(ProcessorChain processorChain) { + this.processorChain = processorChain; + } + + /** + * Maximum number of threads processing URIs at the same time. + */ + int maxToeThreads = 25; + public int getMaxToeThreads() { + return maxToeThreads; + } + public void setMaxToeThreads(int maxToeThreads) { + this.maxToeThreads = maxToeThreads; + if(toePool!=null) { + toePool.setSize(this.maxToeThreads); + } + } + + /** whether to pause, rather than finish, when crawl appears done */ + boolean pauseAtFinish = false; + public boolean getPauseAtFinish() { + return pauseAtFinish; + } + public void setPauseAtFinish(boolean pauseAtFinish) { + this.pauseAtFinish = pauseAtFinish; + } + + /** whether to pause at crawl start */ + boolean pauseAtStart = false; + public boolean getPauseAtStart() { + return pauseAtStart; + } + public void setPauseAtStart(boolean pauseAtStart) { + this.pauseAtStart = pauseAtStart; + } + + /** + * Size in bytes of in-memory buffer to record outbound traffic. One such + * buffer is reserved for every ToeThread. + */ + int recorderOutBufferBytes = 4096; + public int getRecorderOutBufferBytes() { + return recorderOutBufferBytes; + } + public void setRecorderOutBufferBytes(int recorderOutBufferBytes) { + this.recorderOutBufferBytes = recorderOutBufferBytes; + } + + /** + * Size in bytes of in-memory buffer to record inbound traffic. One such + * buffer is reserved for every ToeThread. + */ + int recorderInBufferBytes = 65536; + public int getRecorderInBufferBytes() { + return recorderInBufferBytes; + } + public void setRecorderInBufferBytes(int recorderInBufferBytes) { + this.recorderInBufferBytes = recorderInBufferBytes; + } + + /** + * Period at which to create automatic checkpoints; -1 means + * no auto checkpointing. + */ + int checkpointerPeriod = -1; + public int getCheckpointerPeriod() { + return checkpointerPeriod; + } + public void setCheckpointerPeriod(int checkpointerPeriod) { + this.checkpointerPeriod = checkpointerPeriod; + } + + protected CrawlerLoggerModule loggerModule; + public CrawlerLoggerModule getLoggerModule() { + return this.loggerModule; + } + @Autowired + public void setLoggerModule(CrawlerLoggerModule loggerModule) { + this.loggerModule = loggerModule; + } + + /** + * Messages from the crawlcontroller. + * + * They appear on console. + */ + private final static Logger LOGGER = + Logger.getLogger(CrawlController.class.getName()); + + private transient ToePool toePool; + + // emergency reserve of memory to allow some progress/reporting after OOM + private transient LinkedList reserveMemory; + private static final int RESERVE_BLOCKS = 1; + private static final int RESERVE_BLOCK_SIZE = 6*2^20; // 6MB + + /** + * Crawl exit status. + */ + private transient CrawlStatus sExit; + + public static enum State { + NASCENT, RUNNING, PAUSED, PAUSING, CHECKPOINTING, + STOPPING, FINISHED, PREPARING + } + + transient private State state = State.NASCENT; + + /** + * Checkpointer. + * Knows if checkpoint in progress and what name of checkpoint is. Also runs + * checkpoints. + */ + private Checkpointer checkpointer; + + public CrawlController() { + } + + transient AlertThreadGroup alertThreadGroup; + + public void start() { + // cache AlertThreadGroup for later ToePool launch + AlertThreadGroup atg = AlertThreadGroup.current(); + if(atg!=null) { + alertThreadGroup = atg; + } + + if(isRunning) { + return; + } + this.checkpointer = new Checkpointer( + this, getCheckpointsDir().getFile()); + + sExit = CrawlStatus.FINISHED_ABNORMAL; + + // force creation of DNS Cache now -- avoids CacheCleaner in toe-threads group + // also cap size at 1 (we never wanta cached value; 0 is non-operative) + Lookup.getDefaultCache(DClass.IN).setMaxEntries(1); + + reserveMemory = new LinkedList(); + for(int i = 1; i < RESERVE_BLOCKS; i++) { + reserveMemory.add(new char[RESERVE_BLOCK_SIZE]); + } + isRunning = true; + } + + boolean isRunning = false; + public boolean isRunning() { + return isRunning; + } + + public void stop() { + // TODO: more stop/cleanup? + isRunning = false; + } + + /** + * Send crawl change event to all listeners. + * @param newState State change we're to tell listeners' about. + * @param message Message on state change. + * @see #sendCheckpointEvent(File) for special case event sending + * telling listeners to checkpoint. + */ + @SuppressWarnings("unchecked") + protected void sendCrawlStateChangeEvent(State newState, + CrawlStatus status) { + if(this.state == newState) { + // suppress duplicate state-reports + return; + } + this.state = newState; + CrawlStateEvent event = new CrawlStateEvent(this,newState,status.getDescription()); + appCtx.publishEvent(event); + } + + + // TODO: provide better knowledge/guard against twice-starting + boolean hasStarted = false; + public boolean hasStarted() { + return hasStarted; + } + + /** + * Operator requested crawl begin + */ + public void requestCrawlStart() { + hasStarted = true; + sendCrawlStateChangeEvent(State.PREPARING, CrawlStatus.PREPARING); + frontier.loadSeeds(); + + setupToePool(); + + // A proper exit will change this value. + this.sExit = CrawlStatus.FINISHED_ABNORMAL; + + if (getPauseAtStart()) { + requestCrawlPause(); + } else { + getFrontier().run(); + } + } + + /** + * Called when the last toethread exits. + */ + protected void completeStop() { + LOGGER.fine("Entered complete stop."); + + loggerModule.closeLogFiles(); + this.reserveMemory = null; + if (this.checkpointer != null) { + this.checkpointer.cleanup(); + this.checkpointer = null; + } + if (this.toePool != null) { + this.toePool.cleanup(); + } + this.toePool = null; + + LOGGER.fine("Finished crawl."); + + appCtx.stop(); + // Ok, now we are ready to exit. + sendCrawlStateChangeEvent(State.FINISHED, this.sExit); + } + + synchronized void completePause() { + // Send a notifyAll. At least checkpointing thread may be waiting on a + // complete pause. + notifyAll(); + sendCrawlStateChangeEvent(State.PAUSED, CrawlStatus.PAUSED); + } + + private boolean shouldContinueCrawling() { + Frontier frontier = getFrontier(); + if (frontier.isEmpty()) { + this.sExit = CrawlStatus.FINISHED; + return false; + } + return state == State.RUNNING; + } + + /** + * Request a checkpoint. + * Sets a checkpointing thread running. + * @throws IllegalStateException Thrown if crawl is not in paused state + * (Crawl must be first paused before checkpointing). + */ + public synchronized void requestCrawlCheckpoint() + throws IllegalStateException { + if (this.checkpointer == null) { + return; + } + if (this.checkpointer.isCheckpointing()) { + throw new IllegalStateException("Checkpoint already running."); + } + this.checkpointer.checkpoint(); + } + + /** + * @return True if checkpointing. + */ + public boolean isCheckpointing() { + return this.state == State.CHECKPOINTING; + } + + + /** + * Operator requested for crawl to stop. + */ + public synchronized void requestCrawlStop() { + requestCrawlStop(CrawlStatus.ABORTED); + } + + /** + * Operator requested for crawl to stop. + * @param message + */ + public synchronized void requestCrawlStop(CrawlStatus message) { + if (state == State.STOPPING || state == State.FINISHED || state == State.NASCENT) { + return; + } + if (message == null) { + throw new IllegalArgumentException("Message cannot be null."); + } + this.sExit = message; + beginCrawlStop(); + } + + /** + * Start the process of stopping the crawl. + */ + public void beginCrawlStop() { + LOGGER.fine("Started."); + sendCrawlStateChangeEvent(State.STOPPING, this.sExit); + Frontier frontier = getFrontier(); + if (frontier != null) { + frontier.terminate(); + } + LOGGER.fine("Finished."); + } + + /** + * Stop the crawl temporarly. + */ + public synchronized void requestCrawlPause() { + if (state == State.PAUSING || state == State.PAUSED) { + // Already about to pause + return; + } + sExit = CrawlStatus.WAITING_FOR_PAUSE; + getFrontier().pause(); + sendCrawlStateChangeEvent(State.PAUSING, this.sExit); + // wait for pause to come via frontier changes + } + + /** + * Tell if the controller is paused + * @return true if paused + */ + public boolean isPaused() { + return state == State.PAUSED; + } + + public boolean isPausing() { + return state == State.PAUSING; + } + + public boolean isStateRunning() { + return state == State.RUNNING; + } + + /** + * Resume crawl from paused state + */ + public void requestCrawlResume() { + if (this.toePool == null) { + this.setupToePool(); + } + + if (state != State.PAUSING && state != State.PAUSED && state != State.CHECKPOINTING) { + // Can't resume if not been told to pause or if we're in middle of + // a checkpoint. + return; + } + + Frontier f = getFrontier(); + f.unpause(); + sendCrawlStateChangeEvent(State.RUNNING, CrawlStatus.RUNNING); + } + + /** + * @return Active toe thread count. + */ + public int getActiveToeCount() { + if (toePool == null) { + return 0; + } + return toePool.getActiveToeCount(); + } + + protected void setupToePool() { + toePool = new ToePool(alertThreadGroup,this); + // TODO: make # of toes self-optimizing + toePool.setSize(getMaxToeThreads()); + toePool.waitForAll(); + } + + /** + * @return The number of ToeThreads + * + * @see ToePool#getToeCount() + */ + public int getToeCount() { + return this.toePool == null? 0: this.toePool.getToeCount(); + } + + /** + * @return The ToePool + */ + public ToePool getToePool() { + return toePool; + } + + /** + * Kills a thread. For details see + * {@link org.archive.crawler.framework.ToePool#killThread(int, boolean) + * ToePool.killThread(int, boolean)}. + * @param threadNumber Thread to kill. + * @param replace Should thread be replaced. + * @see org.archive.crawler.framework.ToePool#killThread(int, boolean) + */ + public void killThread(int threadNumber, boolean replace){ + toePool.killThread(threadNumber, replace); + } + + /** + * Evaluate if the crawl should stop because it is finished. + */ + public void checkFinish() { + if(atFinish()) { + beginCrawlStop(); + } + } + + /** + * Evaluate if the crawl should stop because it is finished, + * without actually stopping the crawl. + * + * @return true if crawl is at a finish-possible state + */ + public boolean atFinish() { + return state == State.RUNNING && !shouldContinueCrawling(); + } + + private void readObject(ObjectInputStream stream) + throws IOException, ClassNotFoundException { + this.state = State.PAUSED; + stream.defaultReadObject(); + } + + public void freeReserveMemory() { + if(!reserveMemory.isEmpty()) { + reserveMemory.removeLast(); + System.gc(); + } + } + + /** + * Log to the progress statistics log. + * @param msg Message to write the progress statistics log. + */ + public void logProgressStatistics(final String msg) { + loggerModule.getProgressStats().info(msg); + } + + /** + * @return CrawlController state. + */ + public Object getState() { + return this.state; + } + + public CrawlStatus getCrawlExitStatus() { + return this.sExit; + } + + public String getToeThreadReport() { + StringWriter sw = new StringWriter(); + toePool.reportTo(new PrintWriter(sw)); + return sw.toString(); + } + + public String getToeThreadReportShort() { + return (toePool == null) ? "" : ArchiveUtils.singleLineReport(toePool); + } + + public String getFrontierReport() { + StringWriter sw = new StringWriter(); + try { + getFrontier().reportTo(new PrintWriter(sw)); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return sw.toString(); + } + + public String getFrontierReportShort() { + return ArchiveUtils.singleLineReport(getFrontier()); + } + + /** + * Receive notification from the frontier, in the frontier's own + * manager thread, that the frontier has reached a new state. + * + * @param reachedState the state the frontier has reached + */ + public void noteFrontierState(Frontier.State reachedState) { + switch (reachedState) { + case RUN: + LOGGER.info("Crawl resumed."); + sendCrawlStateChangeEvent(State.RUNNING, CrawlStatus.RUNNING); + case PAUSE: + if (state == State.PAUSING) { + completePause(); + break; + } + if(atFinish()) { // really, "just reached finish" + if (getPauseAtFinish()) { + requestCrawlPause(); + } else { + beginCrawlStop(); + } + break; + } + if(state == State.STOPPING || state == State.FINISHED) { + frontier.requestState(Frontier.State.FINISH); + } + break; + case FINISH: + completeStop(); + break; + default: + // do nothing + } + } +}//EOC diff --git a/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java b/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java new file mode 100644 index 00000000..f5d73dae --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java @@ -0,0 +1,716 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.framework; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.logging.FileHandler; +import java.util.logging.Formatter; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + +import org.apache.commons.collections.ListUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.LineIterator; +import org.apache.commons.lang.StringUtils; +import org.archive.crawler.event.CrawlStateEvent; +import org.archive.crawler.reporting.AlertThreadGroup; +import org.archive.crawler.reporting.CrawlStatSnapshot; +import org.archive.crawler.reporting.StatisticsTracker; +import org.archive.spring.ConfigPath; +import org.archive.spring.ConfigPathConfigurer; +import org.archive.spring.PathSharingContext; +import org.archive.util.ArchiveUtils; +import org.joda.time.DateTime; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.BeanCreationException; +import org.springframework.beans.factory.NoSuchBeanDefinitionException; +import org.springframework.context.ApplicationEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.validation.Errors; +import org.w3c.dom.Document; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +/** + * CrawlJob represents a crawl configuration, including its + * configuration files, instantiated/running ApplicationContext, and + * disk output, potentially across multiple runs. + * + * CrawlJob provides convenience methods for an administrative + * interface to assemble, launch, monitor, and manage crawls. + * + * @contributor gojomo + */ +public class CrawlJob implements Comparable, ApplicationListener{ + File primaryConfig; + PathSharingContext ac; + int launchCount; + DateTime lastLaunch; + AlertThreadGroup alertThreadGroup; + + DateTime xmlOkAt = new DateTime(0L); + Logger jobLogger; + + public CrawlJob(File cxml) { + primaryConfig = cxml; + scanJobLog(); + alertThreadGroup = new AlertThreadGroup(getShortName()); + } + + public File getPrimaryConfig() { + return primaryConfig; + } + public File getJobDir() { + return getPrimaryConfig().getParentFile(); + } + public String getShortName() { + return getJobDir().getName(); + } + public File getJobLog() { + return new File(getJobDir(),"job.log"); + } + + public PathSharingContext getJobContext() { + if(ac==null) { + instantiateContainer(); + } + return ac; + } + + + /** + * Get a logger to a distinguished file, job.log in the job's + * directory, into which job-specific events may be reported. + * + * @return Logger writing to the job-specific log + */ + public Logger getJobLogger() { + if(jobLogger == null) { + jobLogger = Logger.getLogger(getShortName()); + try { + Handler h = new FileHandler(getJobLog().getAbsolutePath(),true); + h.setFormatter(new JobLogFormatter()); + jobLogger.addHandler(h); + } catch (SecurityException e) { + throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return jobLogger; + } + + public DateTime getLastLaunch() { + return lastLaunch; + } + public int getLaunchCount() { + return launchCount; + } + /** + * Refresh knowledge of total launched and last launch by scanning + * the job.log. + */ + protected void scanJobLog() { + File jobLog = getJobLog(); + launchCount = 0; + if(!jobLog.exists()) return; + + try { + LineIterator lines = FileUtils.lineIterator(jobLog); + Pattern launchLine = Pattern.compile("(\\S+) (\\S+) Job launched"); + while(lines.hasNext()) { + String line = lines.nextLine(); + Matcher m = launchLine.matcher(line); + if(m.matches()) { + launchCount++; + lastLaunch = new DateTime(m.group(1)); + } + } + LineIterator.closeQuietly(lines); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + + /** + * Is this job a 'profile' (or template), meaning it may be editted + * or copied to another jobs, but should not be launched. Profiles + * are marked with the convention that their short name + * (job directory name) begins "profile-". + * + * @return true if this job is a 'profile' + */ + public boolean isProfile() { + return primaryConfig.getName().startsWith("profile-"); + } + + // + // writing a basic HTML representation + // + + public void writeHtmlTo(PrintWriter pw) { + writeHtmlTo(pw,"./"); + } + public void writeHtmlTo(PrintWriter pw, String uriPrefix) { + pw.println(""); + if(isRunning()) { + pw.println("ACTIVE; "+getCrawlController().getState()+":"); + } + pw.println(""+getShortName()+""); + if(isProfile()) { + pw.println("(profile)"); + } + pw.println(" " + getLaunchCount() + " launches"); + pw.println("
"); + pw.println(getPrimaryConfig()); + pw.println("
"); + if(lastLaunch!=null) { + pw.println("(last at "+lastLaunch+")"); + } + pw.println(""); + } + + /** + * Is the primary XML config minimally well-formed? + */ + public void checkXML() { + // TODO: suppress check if XML unchanged? job.log when XML changed? + + DateTime testTime = new DateTime(getPrimaryConfig().lastModified()); + Document doc = getDomDocument(getPrimaryConfig()); + // TODO: check for other minimal requirements, like + // presence of a few key components (CrawlController etc.)? + if(doc!=null) { + xmlOkAt = testTime; + } else { + xmlOkAt = new DateTime(0L); + } + + } + + /** + * Read a file to a DOM Document; return null if this isn't possible + * for any reason. + * + * @param f File of XML + * @return org.w3c.dom.Document or null if problems encountered + */ + protected Document getDomDocument(File f) { + try { + DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder docBuilder = docBuilderFactory.newDocumentBuilder(); + return docBuilder.parse(f); + } catch (ParserConfigurationException e) { + return null; + } catch (SAXException e) { + return null; + } catch (IOException e) { + return null; + } + } + + /** + * Is the primary config file legal XML? + * + * @return true if the primary configuration file passed XML testing + */ + public boolean isXmlOk() { + return xmlOkAt.getMillis() >= getPrimaryConfig().lastModified(); + } + + + /** + * Can the configuration yield an assembled ApplicationContext? + */ + public void instantiateContainer() { + checkXML(); + if(ac==null) { + try { + ac = new PathSharingContext(new String[] {"file:"+primaryConfig.getAbsolutePath()},false,null); +// ac = new PathSharingContext(new String[] {primaryConfig.getAbsolutePath()},false,null); + ac.addApplicationListener(this); + ac.refresh(); + getCrawlController(); // trigger NoSuchBeanDefinitionException if no CC + getJobLogger().log(Level.INFO,"Job instantiated"); + } catch (BeansException be) { +// if(ac!=null) { +// ac.close(); +// } + ac = null; + beansException(be); + } + } + } + + /** + * Report a BeansException during instantiation; report chain in + * reverse order (so root cause is first); ignore non-BeansExceptions + * or messages without a useful compact message. + * @param be BeansException + */ + protected void beansException(BeansException be) { + LinkedList beMsgs = new LinkedList(); + Throwable t = be; + while (t!=null) { + if(t instanceof BeansException) { + String msg = shortMessage((BeansException)t); + if(msg!=null) { + beMsgs.add(msg); + } + } + t = t.getCause(); + } + Collections.reverse(beMsgs); + String shortMessage = StringUtils.join(beMsgs,"; "); + + getJobLogger().log(Level.SEVERE,shortMessage,be); + } + + /** + * Return a short useful message for common BeansExceptions. + * @param ex BeansException + * @return String short descriptive message + */ + protected String shortMessage(BeansException ex) { + if(ex instanceof NoSuchBeanDefinitionException) { + NoSuchBeanDefinitionException nsbde = (NoSuchBeanDefinitionException)ex; + return "Missing required bean: " + + (nsbde.getBeanName()!=null ? "\""+nsbde.getBeanName()+"\" " : "") + + (nsbde.getBeanType()!=null ? "\""+nsbde.getBeanType()+"\" " : ""); + } + if(ex instanceof BeanCreationException) { + BeanCreationException bce = (BeanCreationException)ex; + return bce.getBeanName()== null + ? "" + : "Can't create bean '"+bce.getBeanName()+"'"; + } + return ex.getMessage().replace('\n', ' '); + } + + public boolean isContainerOk() { + return ac!=null; + } + + /** + * Does the assembled ApplicationContext self-validate? Any failures + * are reported as WARNING log events in the job log. + * + * TODO: make these severe? + */ + public void validateConfiguration() { + instantiateContainer(); + if(ac==null) { + // fatal errors already encountered and reported + return; + } + ac.validate(); + HashMap allErrors = ac.getAllErrors(); + for(String name : allErrors.keySet()) { + for(Object err : allErrors.get(name).getAllErrors()) { + getJobLogger().log(Level.WARNING,err.toString()); + } + } + } + + /** + * Ddid the ApplicationContext self-validate? + * return true if validation passed without errors + */ + public boolean isContainerValidated() { + if(ac==null) { + return false; + } + HashMap allErrors = ac.getAllErrors(); + return allErrors != null && allErrors.isEmpty(); + } + + // + // Valid job lifecycle operations + // + + /** + * Launch a crawl into 'running' status, assembling if necessary. + * + * (Note the crawl may have been configured to start in a 'paused' + * state.) + */ + public void launch() { + if (isProfile()) { + throw new IllegalArgumentException("Can't launch profile" + this); + } + + if(isRunning()) { + getJobLogger().log(Level.SEVERE,"Can't relaunch running job"); + return; + } else { + CrawlController cc = getCrawlController(); + if(cc!=null && cc.hasStarted()) { + getJobLogger().log(Level.SEVERE,"Can't relaunch previously-launched assembled job"); + return; + } + } + + validateConfiguration(); + if(!isContainerValidated()) { + getJobLogger().log(Level.SEVERE,"Can't launch problem configuration"); + return; + } + + //final String job = changeState(j, ACTIVE); + + // this temporary thread ensures all crawl-created threads + // land in the AlertThreadGroup, to assist crawl-wide + // logging/alerting + alertThreadGroup = new AlertThreadGroup(getShortName()); + alertThreadGroup.addLogger(getJobLogger()); + Thread launcher = new Thread(alertThreadGroup, getShortName()) { + public void run() { + startContext(); + } + }; + launcher.start(); + + try { + launcher.join(); + } catch (InterruptedException e) { + // do nothing + } + + if(ac==null) { + // unlaunchable + return; + } + getCrawlController().requestCrawlStart(); + getJobLogger().log(Level.INFO,"Job launched"); + scanJobLog(); + } + + /** + * Start the context, catching and reporting any BeansExceptions. + */ + protected void startContext() { + try { + ac.start(); + } catch (BeansException be) { + ac.close(); + ac = null; + beansException(be); + } catch (Exception e) { + e.printStackTrace(System.err); + getJobLogger().log(Level.SEVERE,e.getMessage(),e); + try { + ac.close(); + } catch (Exception e2) { + e2.printStackTrace(System.err); + } finally { + ac = null; + } + } + } + + /** + * Sort for reverse-chronological listing. + * + * @see java.lang.Comparable#compareTo(java.lang.Object) + */ + public int compareTo(CrawlJob o) { + // prefer reverse-chronological ordering + return -((Long)getLastActivityTime()).compareTo(o.getLastActivityTime()); + } + + public long getLastActivityTime() { + return Math.max(getPrimaryConfig().lastModified(), getJobLog().lastModified()); + } + + public boolean isRunning() { + return this.ac != null && this.ac.isActive() && this.ac.isRunning(); + } + + public CrawlController getCrawlController() { + if(ac==null) { + return null; + } + return (CrawlController) ac.getBean("crawlController"); + } + + public boolean isPausable() { + CrawlController cc = getCrawlController(); + if(cc==null) { + return false; + } + return cc.isStateRunning(); + } + + public boolean isUnpausable() { + CrawlController cc = getCrawlController(); + if(cc==null) { + return false; + } + return cc.isPaused() || cc.isPausing(); + } + + /** + * Ensure a fresh start for any configuration changes or relaunches, + * by stopping and discarding an existing ApplicationContext. + */ + public void reset() { + if(ac!=null) { + CrawlController cc = getCrawlController(); + if(cc!=null) { + cc.requestCrawlStop(); + // TODO: wait for stop? + } + if(ac.isRunning()) { + ac.stop(); + } + ac = null; + } + xmlOkAt = new DateTime(0); + getJobLogger().log(Level.INFO,"Job instance discarded"); + } + + /** + * Formatter for job.log + */ + public class JobLogFormatter extends Formatter { + @Override + public String format(LogRecord record) { + StringBuilder sb = new StringBuilder(); + sb + .append(new DateTime(record.getMillis())) + .append(" ") + .append(record.getLevel()) + .append(" ") + .append(record.getMessage()) + .append("\n"); + return sb.toString(); + } + } + + /** + * Return all config files included via 'import' statements in the + * primary config (or other included configs). + * + * @param xml File to examine + * @return List of all transitively-imported Files + */ + @SuppressWarnings("unchecked") + public List getImportedConfigs(File xml) { + List imports = new LinkedList(); + Document doc = getDomDocument(xml); + if(doc==null) { + return ListUtils.EMPTY_LIST; + } + NodeList importElements = doc.getElementsByTagName("import"); + for(int i = 0; i < importElements.getLength(); i++) { + File imported = new File( + getJobDir(), + importElements.item(i).getAttributes().getNamedItem("resource").getTextContent()); + imports.add(imported); + imports.addAll(getImportedConfigs(imported)); + } + return imports; + } + + /** + * Return all known ConfigPaths, as an aid to viewing or editting. + * + * @return all ConfigPaths known to the ApplicationContext, in a + * map by name, or an empty map if no ApplicationContext + */ + @SuppressWarnings("unchecked") + public Map getConfigPaths() { + if(ac==null) { + return MapUtils.EMPTY_MAP; + } + ConfigPathConfigurer cpc = + (ConfigPathConfigurer)ac.getBean("configPathConfigurer"); + return cpc.getPaths(); + } + + /** + * Compute a path relative to the job directory for all contained + * files, or null if the File is not inside the job directory. + * + * @param f File + * @return path relative to the job directory, or null if File not + * inside job dir + */ + public String jobDirRelativePath(File f) { + try { + String filePath = f.getCanonicalPath(); + String jobPath = getJobDir().getCanonicalPath(); + if(filePath.startsWith(jobPath)) { + return filePath.substring(jobPath.length()).replace(File.separatorChar, '/'); + } + } catch (IOException e) { + getJobLogger().log(Level.WARNING,"bad file: "+f); + } + return null; + } + + /** + * Log note of all ApplicationEvents. + * + * @see org.springframework.context.ApplicationListener#onApplicationEvent(org.springframework.context.ApplicationEvent) + */ + public void onApplicationEvent(ApplicationEvent event) { + if(event instanceof CrawlStateEvent) { + getJobLogger().log(Level.INFO, ((CrawlStateEvent)event).getState().toString()); + } + } + + /** + * Is this launchable? (Has CrawlController and not yet been launched?) + * @return true if launchable + */ + public boolean isLaunchable() { + CrawlController cc = getCrawlController(); + if(cc==null) { + return true; + } + return !cc.hasStarted(); + } + + public int getAlertCount() { + return alertThreadGroup.getAlertCount(); + } + + protected StatisticsTracker getStats() { + CrawlController cc = getCrawlController(); + return cc!=null ? cc.getStatisticsTracker() : null; + } + + public Object rateReport() { + StatisticsTracker stats = getStats(); + if(stats==null) { + return "n/a"; + } + CrawlStatSnapshot snapshot = stats.getSnapshot(); + StringBuilder sb = new StringBuilder(); + sb + .append(ArchiveUtils.doubleToString(snapshot.currentDocsPerSecond,2)) + .append(" URIs/sec (") + .append(ArchiveUtils.doubleToString(snapshot.docsPerSecond,2)) + .append(" avg); ") + .append(snapshot.currentKiBPerSec) + .append(" KB/sec (") + .append(snapshot.totalKiBPerSec) + .append(" avg)"); + return sb.toString(); + } + + public Object loadReport() { + StatisticsTracker stats = getStats(); + if(stats==null) { + return "n/a"; + } + CrawlStatSnapshot snapshot = stats.getSnapshot(); + StringBuilder sb = new StringBuilder(); + sb + .append(snapshot.busyThreads) + .append(" active of ") + .append(stats.threadCount()) + .append(" threads; ") + .append(ArchiveUtils.doubleToString(snapshot.congestionRatio,2)) + .append(" congestion ratio; ") + .append(snapshot.deepestUri) + .append(" deepest queue; ") + .append(snapshot.averageDepth) + .append(" average depth"); + return sb.toString(); + } + + public String uriTotalsReport() { + StatisticsTracker stats = getStats(); + if(stats==null) { + return "n/a"; + } + CrawlStatSnapshot snapshot = stats.getSnapshot(); + long downloaded = snapshot.downloadedUriCount; + long total = snapshot.totalCount(); + long queued = snapshot.queuedUriCount; + StringBuilder sb = new StringBuilder(64); + sb + .append(downloaded) + .append(" downloaded + ") + .append(queued) + .append(" queued = ") + .append(total) + .append(" total"); + return sb.toString(); + } + + public String sizeTotalsReport() { + StatisticsTracker stats = getStats(); + if(stats==null) { + return "n/a"; + } + return stats.crawledBytesSummary(); + } + + public String elapsedReport() { + StatisticsTracker stats = getStats(); + if(stats==null) { + return "n/a"; + } + long timeElapsed = stats.getCrawlElapsedTime(); + return ArchiveUtils.formatMillisecondsToConventional(timeElapsed,false); + } + + public String threadReport() { + CrawlController cc = getCrawlController(); + if(cc==null) { + return "n/a"; + } + return cc.getToeThreadReportShort(); + } + + public String frontierReport() { + CrawlController cc = getCrawlController(); + if(cc==null) { + return "n/a"; + } + return cc.getFrontierReportShort(); + } + + public void terminate() { + getCrawlController().requestCrawlStop(); + } + +}//EOC diff --git a/engine/src/main/java/org/archive/crawler/framework/CrawlLimitEnforcer.java b/engine/src/main/java/org/archive/crawler/framework/CrawlLimitEnforcer.java new file mode 100644 index 00000000..da19d63b --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/framework/CrawlLimitEnforcer.java @@ -0,0 +1,102 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.framework; + +import org.archive.crawler.event.StatSnapshotEvent; +import org.archive.crawler.reporting.CrawlStatSnapshot; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationEvent; +import org.springframework.context.ApplicationListener; + +/** + * Bean to enforce limits on the size of a crawl in URI count, + * byte count, or elapsed time. Fires off the StatSnapshotEvent, + * so only checks at the interval (configured in StatisticsTracker) + * of those events. + * + * @contributor gojomo + */ +public class CrawlLimitEnforcer implements ApplicationListener { + + /** + * Maximum number of bytes to download. Once this number is exceeded + * the crawler will stop. A value of zero means no upper limit. + */ + long maxBytesDownload = 0L; + public long getMaxBytesDownload() { + return maxBytesDownload; + } + public void setMaxBytesDownload(long maxBytesDownload) { + this.maxBytesDownload = maxBytesDownload; + } + + /** + * Maximum number of documents to download. Once this number is exceeded the + * crawler will stop. A value of zero means no upper limit. + */ + long maxDocumentsDownload = 0L; + public long getMaxDocumentsDownload() { + return maxDocumentsDownload; + } + public void setMaxDocumentsDownload(long maxDocumentsDownload) { + this.maxDocumentsDownload = maxDocumentsDownload; + } + + /** + * Maximum amount of time to crawl (in seconds). Once this much time has + * elapsed the crawler will stop. A value of zero means no upper limit. + */ + long maxTimeSeconds = 0L; + public long getMaxTimeSeconds() { + return maxTimeSeconds; + } + public void setMaxTimeSeconds(long maxTimeSeconds) { + this.maxTimeSeconds = maxTimeSeconds; + } + + protected CrawlController controller; + public CrawlController getCrawlController() { + return this.controller; + } + @Autowired + public void setCrawlController(CrawlController controller) { + this.controller = controller; + } + + public void onApplicationEvent(ApplicationEvent event) { + if(event instanceof StatSnapshotEvent) { + CrawlStatSnapshot snapshot = ((StatSnapshotEvent)event).getSnapshot(); + checkForLimitsExceeded(snapshot); + } + } + + protected void checkForLimitsExceeded(CrawlStatSnapshot snapshot) { + if (maxBytesDownload > 0 && snapshot.bytesProcessed >= maxBytesDownload) { + controller.requestCrawlStop(CrawlStatus.FINISHED_DATA_LIMIT); + } else if (maxDocumentsDownload > 0 + && snapshot.downloadedUriCount >= maxDocumentsDownload) { + controller.requestCrawlStop(CrawlStatus.FINISHED_DOCUMENT_LIMIT); + } else if (maxTimeSeconds > 0 + && snapshot.elapsedMilliseconds >= maxTimeSeconds * 1000) { + controller.requestCrawlStop(CrawlStatus.FINISHED_TIME_LIMIT); + } + } + +} diff --git a/engine/src/main/java/org/archive/crawler/framework/CrawlStatus.java b/engine/src/main/java/org/archive/crawler/framework/CrawlStatus.java new file mode 100644 index 00000000..e964cf53 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/framework/CrawlStatus.java @@ -0,0 +1,87 @@ +package org.archive.crawler.framework; + +public enum CrawlStatus { + + /** Inital value. May not be ready to run/incomplete. */ + CREATED("Created"), + + /** Job has been successfully submitted to a CrawlJobHandler. */ + PENDING("Pending"), + + /** Job is being crawled. */ + RUNNING("Running"), + + /** Job was deleted by user, will not be displayed in UI. */ + DELETED("Deleted"), + + /** Job was terminted by user input while crawling. */ + ABORTED("Finished - Ended by operator"), + + /** Something went very wrong. */ + FINISHED_ABNORMAL("Finished - Abnormal exit from crawling"), + + /** Job finished normally having completed its crawl. */ + FINISHED("Finished"), + + /** Job finished normally when the specified timelimit was hit. */ + FINISHED_TIME_LIMIT("Finished - Timelimit hit"), + + + /** + * Job finished normally when the specifed amount of data (MB) had been + * downloaded + */ + FINISHED_DATA_LIMIT("Finished - Maximum amount of data limit hit"), + + + /** + * Job finished normally when the specified number of documents had been + * fetched. + */ + FINISHED_DOCUMENT_LIMIT("Finished - Maximum number of documents limit hit"), + + FINISHED_WRITE_LIMIT("Finished - Maximum bytes written"), + + /** + * Job is going to be temporarly stopped after active threads are finished. + */ + WAITING_FOR_PAUSE("Pausing - Waiting for threads to finish"), + + + /** Job was temporarly stopped. State is kept so it can be resumed */ + PAUSED("Paused"), + + + /** + * Job is being checkpointed. When finished checkpointing, job is set + * back to STATUS_PAUSED (Job must be first paused before checkpointing + * will run). + */ + CHECKPOINTING("Checkpointing"), + + + /** Job could not be launced due to an InitializationException */ + MISCONFIGURED("Could not launch job - Fatal InitializationException"), + + + /** Job is actually a profile */ + PROFILE("Profile"), + + + /** Prepared. */ + PREPARING("Preparing"); + + + + final public String desc; + + CrawlStatus(String desc) { + this.desc = desc; + } + + + public String getDescription() { + return desc; + } + +} diff --git a/engine/src/main/java/org/archive/crawler/framework/Engine.java b/engine/src/main/java/org/archive/crawler/framework/Engine.java new file mode 100644 index 00000000..e39f7b53 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/framework/Engine.java @@ -0,0 +1,259 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.framework; + +import java.io.File; +import java.io.FileFilter; +import java.io.FilenameFilter; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.FileFilterUtils; +import org.archive.util.ArchiveUtils; + +/** + * Implementation for Engine. Jobs and profiles are stored in a + * directory called the jobsDir. The jobs are contained as subdirectories of + * jobDir. + * + * @contributor pjack + * @contributor gojomo + */ +public class Engine { + private static final long serialVersionUID = 4L; + + final public static String LOGS_DIR_NAME = "logs subdirectory"; + final public static String REPORTS_DIR_NAME = "reports subdirectory"; + + final private static Logger LOGGER = + Logger.getLogger(Engine.class.getName()); + + /** directory where job directores are expected */ + protected File jobsDir; + /** map of job short names -> CrawlJob instances */ + protected HashMap jobConfigs = new HashMap(); + + public Engine(File jobsDir) { + this.jobsDir = jobsDir; + this.jobsDir.mkdirs(); + + findJobConfigs(); + // TODO: cleanup any cruft from improperly ended jobs + } + + /** + * Find all job configurations in the usual place -- subdirectories + * of the jobs directory with files ending '.cxml'. + */ + public void findJobConfigs() { + // TODO: allow other places/paths to be scanned/added as well? + + // remove crawljobs whose directories have disappeared + // TODO: try a more delicate cleanup; eg: if appCtx exists? + for(String jobName: jobConfigs.keySet().toArray(new String[0])) { + CrawlJob cj = jobConfigs.get(jobName); + if(!cj.getJobDir().exists()) { + jobConfigs.remove(jobName); + } + } + + // discover any new job directories + for (File dir : jobsDir.listFiles(new FileFilter(){ + public boolean accept(File pathname) { + return pathname.isDirectory(); + }})) { + for (File cxml : dir.listFiles(new FilenameFilter() { + public boolean accept(File dir, String name) { + return name.endsWith(".cxml"); + }})) { + try { + CrawlJob cj = new CrawlJob(cxml); + if(!jobConfigs.containsKey(cj.getShortName())) { + jobConfigs.put(cj.getShortName(),cj); + } + } catch (IllegalArgumentException iae) { + LOGGER.log(Level.WARNING,"bad cxml: "+cxml,iae); + } + } + } + } + + public Map getJobConfigs() { + return jobConfigs; + } + + + + /** + * Copy a job to a new location, possibly making a job + * a profile or a profile a runnable job. + * + * @param orig CrawlJob representing source + * @param destDir File location destination + * @param asProfile true if destination should become a profile + * @throws IOException + */ + public synchronized void copy(CrawlJob orig, File destDir, boolean asProfile) + throws IOException { + destDir.mkdirs(); + if(destDir.list().length>0) { + throw new IOException("destination dir not empty"); + } + File srcDir = orig.getPrimaryConfig().getParentFile(); + + // FIXME: Add option for only copying history DB + // FIXME: Don't hardcode these names + // FIXME: (?) copy any referenced file (ConfigFile/ConfigPath), + // even outside the job directory? + + // copy all simple files except the 'job.log' and its '.lck' (if any) + FileUtils.copyDirectory(srcDir, destDir, + FileFilterUtils.andFileFilter( + FileFilterUtils.fileFileFilter(), + FileFilterUtils.notFileFilter( + FileFilterUtils.prefixFileFilter("job.log")))); + + // ...and all contents of 'resources' subdir... + File srcResources = new File(srcDir, "resources"); + if (srcResources.isDirectory()) { + FileUtils.copyDirectory(srcResources, new File(destDir, "resources")); + } + + File newPrimaryConfig = new File(destDir, orig.getPrimaryConfig().getName()); + if(asProfile) { + if(!orig.isProfile()) { + // rename cxml to have 'profile-' prefix + FileUtils.moveFile( + newPrimaryConfig, + new File(destDir, "profile-"+newPrimaryConfig.getName())); + } + } else { + if(orig.isProfile()) { + // rename cxml to remove 'profile-' prefix + FileUtils.moveFile( + newPrimaryConfig, + new File(destDir, newPrimaryConfig.getName().substring(8))); + } + } + findJobConfigs(); + } + + /** + * Copy a job to a new location, possibly making a job + * a profile or a profile a runnable job. + * + * @param cj CrawlJob representing source + * @param copyTo String location destination; interpreted relative to jobsDir + * @param asProfile true if destination should become a profile + * @throws IOException + */ + public void copy(CrawlJob cj, String copyTo, boolean asProfile) throws IOException { + File dest = new File(copyTo); + if(!dest.isAbsolute()) { + dest = new File(jobsDir,copyTo); + } + copy(cj,dest,asProfile); + } + + public String getHeritrixVersion(){ + return ArchiveUtils.VERSION; + } + + public synchronized void deleteJob(CrawlJob job) throws IOException { + FileUtils.deleteDirectory(job.getJobDir()); + } + + public void requestLaunch(String shortName) { + jobConfigs.get(shortName).launch(); + } + + public CrawlJob getJob(String shortName) { + if(!jobConfigs.containsKey(shortName)) { + // try a rescan if not already present + findJobConfigs(); + } + return jobConfigs.get(shortName); + } + + public File getJobsDir() { + return jobsDir; + } + + public String heapReport() { + long totalMemory = Runtime.getRuntime().totalMemory(); + long freeMemory = Runtime.getRuntime().freeMemory(); + long maxMemory = Runtime.getRuntime().maxMemory(); + StringBuilder sb = new StringBuilder(64); + sb + .append((totalMemory-freeMemory)/1024) + .append(" KiB used; ") + .append(totalMemory/1024) + .append(" KiB current heap; ") + .append(maxMemory/1024) + .append(" KiB max heap"); + return sb.toString(); + } + + public void shutdown() { + // TODO stop everything + for(CrawlJob job : jobConfigs.values()) { + if(job.isRunning()) { + job.terminate(); + } + } + waitForNoRunningJobs(0); + } + + /** + * Wait for all jobs to be in non-running state, or until timeout + * (given in ms) elapses. Use '0' for no timeout (wait as long as + * necessary. + * + * @param timeout + * @return true if timeout occurred and a job is (possibly) still running + */ + public boolean waitForNoRunningJobs(long timeout) { + long startTime = System.currentTimeMillis(); + // wait for all jobs to not be running + outer: while(true) { + if(timeout>0 && (startTime+timeout)>System.currentTimeMillis()) { + return true; + } + try { + Thread.sleep(500); + } catch (InterruptedException e) { + break; + } + for(CrawlJob job : jobConfigs.values()) { + if(job.isRunning()) { + continue outer; + } + } + break; + } + return false; + } + + +} \ No newline at end of file diff --git a/engine/src/main/java/org/archive/crawler/framework/Frontier.java b/engine/src/main/java/org/archive/crawler/framework/Frontier.java new file mode 100644 index 00000000..d839a7dc --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/framework/Frontier.java @@ -0,0 +1,510 @@ +/* Frontier + * + * $Id$ + * + * Created on Mar 29, 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.crawler.framework; + +import java.io.IOException; + +import javax.management.openmbean.CompositeData; + +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.crawler.frontier.FrontierJournal; +import org.archive.modules.deciderules.DecideRule; +import org.archive.modules.fetcher.FetchStats; +import org.archive.net.UURI; +import org.archive.util.MultiReporter; +import org.json.JSONException; + + +/** + * An interface for URI Frontiers. + * + *

A URI Frontier is a pluggable module in Heritrix that maintains the + * internal state of the crawl. This includes (but is not limited to): + *

    + *
  • What URIs have been discovered + *
  • What URIs are being processed (fetched) + *
  • What URIs have been processed + *
  • In what order unprocessed URIs will be processed + *
+ * + *

The Frontier is also responsible for enforcing any politeness restrictions + * that may have been applied to the crawl. Such as limiting simultaneous + * connection to the same host, server or IP number to 1 (or any other fixed + * amount), delays between connections etc. + * + *

A URIFrontier is created by the + * {@link org.archive.crawler.framework.CrawlController CrawlController} which + * is in turn responsible for providing access to it. Most significant among + * those modules interested in the Frontier are the + * {@link org.archive.crawler.framework.ToeThread ToeThreads} who perform the + * actual work of processing a URI. + * + *

The methods defined in this interface are those required to get URIs for + * processing, report the results of processing back (ToeThreads) and to get + * access to various statistical data along the way. The statistical data is + * of interest to {@link org.archive.crawler.framework.StatisticsTracker + * Statistics Tracking} modules. A couple of additional methods are provided + * to be able to inspect and manipulate the Frontier at runtime. + * + *

The statistical data exposed by this interface is: + *

    + *
  • {@link #discoveredUriCount() Discovered URIs} + *
  • {@link #queuedUriCount() Queued URIs} + *
  • {@link #finishedUriCount() Finished URIs} + *
  • {@link #succeededFetchCount() Successfully processed URIs} + *
  • {@link #failedFetchCount() Failed to process URIs} + *
  • {@link #disregardedUriCount() Disregarded URIs} + *
  • {@link #totalBytesWritten() Total bytes written} + *
+ * + *

In addition the frontier may optionally implement an interface that + * exposes information about hosts. + * + *

Furthermore any implementation of the URI Frontier should trigger + * {@link org.archive.crawler.event.CrawlURIDispositionEvent + * CrawlURIDispostionEvents} on the ApplicationContext to allow + * statistics modules or other interested observers to collect info + * about each completed URI's processing. + * + *

All URI Frontiers inherit from + * {@link org.archive.crawler.settings.ModuleType ModuleType} + * and therefore creating settings follows the usual pattern of pluggable modules + * in Heritrix. + * + * @author Gordon Mohr + * @author Kristinn Sigurdsson + * + * @see org.archive.crawler.framework.CrawlController + * @see org.archive.crawler.framework.CrawlController#fireCrawledURIDisregardEvent(CrawlURI) + * @see org.archive.crawler.framework.CrawlController#fireCrawledURIFailureEvent(CrawlURI) + * @see org.archive.crawler.framework.CrawlController#fireCrawledURINeedRetryEvent(CrawlURI) + * @see org.archive.crawler.framework.CrawlController#fireCrawledURISuccessfulEvent(CrawlURI) + * @see org.archive.crawler.framework.StatisticsTracker + * @see org.archive.crawler.framework.ToeThread + * @see org.archive.crawler.settings.ModuleType + */ +public interface Frontier extends MultiReporter { + + /** + * Get the next URI that should be processed. If no URI becomes availible + * during the time specified null will be returned. + * + * @return the next URI that should be processed. + * @throws InterruptedException + */ + CrawlURI next() throws InterruptedException; + + /** + * Returns true if the frontier contains no more URIs to crawl. + * + *

That is to say that there are no more URIs either currently availible + * (ready to be emitted), URIs belonging to deferred hosts or pending URIs + * in the Frontier. Thus this method may return false even if there is no + * currently availible URI. + * + * @return true if the frontier contains no more URIs to crawl. + */ + boolean isEmpty(); + + /** + * Schedules a CrawlURI. + * + *

This method accepts one URI and schedules it immediately. This has + * nothing to do with the priority of the URI being scheduled. Only that + * it will be placed in it's respective queue at once. For priority + * scheduling see {@link CrawlURI#setSchedulingDirective(int)} + * + *

This method should be synchronized in all implementing classes. + * + * @param caURI The URI to schedule. + * + * @see CrawlURI#setSchedulingDirective(int) + */ + public void schedule(CrawlURI caURI); + + /** + * Report a URI being processed as having finished processing. + * + *

ToeThreads will invoke this method once they have completed work on + * their assigned URI. + * + *

This method is synchronized. + * + * @param cURI The URI that has finished processing. + */ + public void finished(CrawlURI cURI); + + /** + * Number of discovered URIs. + * + *

That is any URI that has been confirmed be within 'scope' + * (i.e. the Frontier decides that it should be processed). This + * includes those that have been processed, are being processed + * and have finished processing. Does not include URIs that have + * been 'forgotten' (deemed out of scope when trying to fetch, + * most likely due to operator changing scope definition). + * + *

Note: This only counts discovered URIs. Since the same + * URI can (at least in most frontiers) be fetched multiple times, this + * number may be somewhat lower then the combined queued, + * in process and finished items combined due to duplicate + * URIs being queued and processed. This variance is likely to be especially + * high in Frontiers implementing 'revist' strategies. + * + * @return Number of discovered URIs. + */ + public long discoveredUriCount(); + + /** + * Number of URIs queued up and waiting for processing. + * + *

This includes any URIs that failed but will be retried. Basically this + * is any discovered URI that has not either been processed or is + * being processed. The same discovered URI can be queued multiple times. + * + * @return Number of queued URIs. + */ + public long queuedUriCount(); + + /** + * Ordinal position of the 'deepest' URI eligible + * for crawling. Essentially, the length of the longest + * frontier internal queue. + * + * @return long URI count to deepest URI + */ + public long deepestUri(); // aka longest queue + + /** + * Average depth of the last URI in all eligible queues. + * That is, the average length of all eligible queues. + * + * @return long average depth of last URIs in queues + */ + public long averageDepth(); // aka average queue length + + /** + * Ratio of number of threads that would theoretically allow + * maximum crawl progress (if each was as productive as current + * threads), to current number of threads. + * + * @return float congestion ratio + */ + public float congestionRatio(); // multiple of threads needed for max progress + + /** + * Number of URIs that have finished processing. + * + *

Includes both those that were processed successfully and failed to be + * processed (excluding those that failed but will be retried). Does not + * include those URIs that have been 'forgotten' (deemed out of scope when + * trying to fetch, most likely due to operator changing scope definition). + * + * @return Number of finished URIs. + */ + public long finishedUriCount(); + + /** + * Number of successfully processed URIs. + * + *

Any URI that was processed successfully. This includes URIs that + * returned 404s and other error codes that do not originate within the + * crawler. + * + * @return Number of successfully processed URIs. + */ + public long succeededFetchCount(); + + /** + * Number of URIs that failed to process. + * + *

URIs that could not be processed because of some error or failure in + * the processing chain. Can include failure to acquire prerequisites, to + * establish a connection with the host and any number of other problems. + * Does not count those that will be retried, only those that have + * permenantly failed. + * + * @return Number of URIs that failed to process. + */ + public long failedFetchCount(); + + /** + * Number of URIs that were scheduled at one point but have been + * disregarded. + * + *

Counts any URI that is scheduled only to be disregarded + * because it is determined to lie outside the scope of the crawl. Most + * commonly this will be due to robots.txt exclusions. + * + * @return The number of URIs that have been disregarded. + */ + public long disregardedUriCount(); + + /** + * Total number of bytes contained in all URIs that have been processed. + * + * @return The total amounts of bytes in all processed URIs. + * @deprecated misnomer; consult StatisticsTracker instead + */ + public long totalBytesWritten(); + + /** + * Load URIs from a file, for scheduling and/or considered-included + * status (if from a recovery log). + * + *

The 'params' Map describes the source file to use and options + * in effect regarding its format and handling. Significant keys + * are: + * + *

"path": full path to source file. If the path ends '.gz', it + * will be considered to be GZIP compressed. + *

"format": one of "onePer", "crawlLog", or "recoveryLog" + *

"forceRevisit": if non-null, URIs will be force-scheduled even + * if already considered included + *

"scopeSchedules": if non-null, any URI imported be checked + * against the frontier's configured scope before scheduling + * + *

If the "format" is "recoveryLog", 7 more keys are significant: + * + *

"includeSuccesses": if non-null, success lines ("Fs") in the log + * will be considered-included. (Usually, this is the aim of + * a recovery-log import.) + *

"includeFailures": if non-null, failure lines ("Ff") in the log + * will be considered-included. (Sometimes, this is desired.) + *

"includeScheduleds": If non-null, scheduled lines ("F+") in the + * log will be considered-included. (Atypical, but an option for + * completeness.) + *

"scopeIncludes": if non-null, any of the above will be checked + * against the frontier's configured scope before consideration + * + *

"scheduleSuccesses": if non-null, success lines ("Fs") in the log + * will be schedule-attempted. (Atypical, as all successes + * are preceded by "F+" lines.) + *

"scheduleFailures": if non-null, failure lines ("Ff") in the log + * will be schedule-attempted. (Atypical, as all failures + * are preceded by "F+" lines.) + *

"scheduleScheduleds": if non-null, scheduled lines ("F+") in the + * log will be considered-included. (Usually, this is the aim of a + * recovery-log import.) + * + * TODO: add parameter for auto-unpause-at-good-time + * + * @param params Map describing source file and options as above + * @throws IOException If problems occur reading file. + * @throws JSONException + */ + public void importURIs( + String params) + throws IOException; + + /** + * Get a URIFrontierMarker initialized with the given + * regular expression at the 'start' of the Frontier. + * @param regexpr The regular expression that URIs within the frontier must + * match to be considered within the scope of this marker + * @param inCacheOnly If set to true, only those URIs within the frontier + * that are stored in cache (usually this means in memory + * rather then on disk, but that is an implementation + * detail) will be considered. Others will be entierly + * ignored, as if they dont exist. This is usefull for quick + * peeks at the top of the URI list. + * @return A URIFrontierMarker that is set for the 'start' of the frontier's + * URI list. + */ +// public FrontierMarker getInitialMarker(String regexpr, +// boolean inCacheOnly); + + /** + * Returns a list of all uncrawled URIs starting from a specified marker + * until numberOfMatches is reached. + * + *

Any encountered URI that has not been successfully crawled, terminally + * failed, disregarded or is currently being processed is included. As + * there may be duplicates in the frontier, there may also be duplicates + * in the report. Thus this includes both discovered and pending URIs. + * + *

The list is a set of strings containing the URI strings. If verbose is + * true the string will include some additional information (path to URI + * and parent). + * + *

The URIFrontierMarker will be advanced to the position at + * which it's maximum number of matches found is reached. Reusing it for + * subsequent calls will thus effectively get the 'next' batch. Making + * any changes to the frontier can invalidate the marker. + * + *

While the order returned is consistent, it does not have any + * explicit relation to the likely order in which they may be processed. + * + *

Warning: It is unsafe to make changes to the frontier while + * this method is executing. The crawler should be in a paused state before + * invoking it. + * + * @param marker + * A marker specifing from what position in the Frontier the + * list should begin. + * @param numberOfMatches + * how many URIs to add at most to the list before returning it + * @param verbose + * if set to true the strings returned will contain additional + * information about each URI beyond their names. + * @return a list of all pending URIs falling within the specification + * of the marker + * @throws InvalidFrontierMarkerException when the + * URIFronterMarker does not match the internal + * state of the frontier. Tolerance for this can vary + * considerably from one URIFrontier implementation to the next. + * @see FrontierMarker + * @see #getInitialMarker(String, boolean) + */ + public CompositeData getURIsList( + String marker, + int numberOfMatches, + String regex, + boolean verbose); + + /** + * Delete any URI that matches the given regular expression from the list + * of discovered and pending URIs. This does not prevent them from being + * rediscovered. + * + *

Any encountered URI that has not been successfully crawled, terminally + * failed, disregarded or is currently being processed is considered to be + * a pending URI. + * + *

Warning: It is unsafe to make changes to the frontier while + * this method is executing. The crawler should be in a paused state before + * invoking it. + * + * @param match A regular expression, any URIs that matches it will be + * deleted. + * @return The number of URIs deleted + */ + public long deleteURIs( + String queueRegex, + String match); + + /** + * Notify Frontier that a CrawlURI has been deleted outside of the + * normal next()/finished() lifecycle. + * + * @param curi Deleted CrawlURI. + */ + public void deleted(CrawlURI curi); + + /** + * Notify Frontier that it should consider the given UURI as if + * already scheduled. + * + * @param u UURI instance to add to the Already Included set. + */ + public void considerIncluded(UURI u); + + /** + * Notify Frontier that it should not release any URIs, instead + * holding all threads, until instructed otherwise. + */ + public void pause(); + + /** + * Resumes the release of URIs to crawl, allowing worker + * ToeThreads to proceed. + */ + public void unpause(); + + /** + * Notify Frontier that it should end the crawl, giving + * any worker ToeThread that askss for a next() an + * EndedException. + */ + public void terminate(); + + /** + * @return Return the instance of {@link FrontierJournal} that + * this Frontier is using. May be null if no journaling. + */ + public FrontierJournal getFrontierJournal(); + + /** + * @param cauri CrawlURI for which we're to calculate and + * set class key. + * @return Classkey for cauri. + */ + public String getClassKey(CrawlURI cauri); + + /* + * Return the internally-configured crawl 'scope' (rules for + * deciding whether a URI is crawled or not). + */ + public DecideRule getScope(); + + /** + * Request that the Frontier load (or reload) crawl seeds, + * typically by contacting the Scope. + */ + public void loadSeeds(); + + /** + * Request that Frontier allow crawling to begin. Usually + * just unpauses Frontier, if paused. + */ + public void run(); + + /** + * Get the 'frontier group' (usually queue) for the given + * CrawlURI. + * @param curi CrawlURI to find matching group + * @return FrontierGroup for the CrawlURI + */ + public FrontierGroup getGroup(CrawlURI curi); + + /** + * Generic interface representing the internal groupings + * of a Frontier's URIs -- usually queues. Currently only + * offers the HasCrawlSubstats interface. + */ + public interface FrontierGroup + extends FetchStats.HasFetchStats, FetchStats.CollectsFetchStats { + } + + /** + * Request the Frontier reach the given state as soon as possible. (Only + * when a later notification is given the CrawlController has the state + * actually been reached.) + * + * @param target Frontier.State to pursue + */ + public void requestState(State target); + + /** + * Enumeration of possible target states. + */ + public enum State { + RUN, // juggle/prioritize/emit; usual state + HOLD, // enter a consistent, stable, checkpointable state ASAP + PAUSE, // enter a stable state where no URIs are in-progress; unlike + // HOLD requires all in-process URIs to complete + FINISH }; // end and cleanup; may not return to any other state after + // this state is requested/reached +} diff --git a/engine/src/main/java/org/archive/crawler/framework/Scoper.java b/engine/src/main/java/org/archive/crawler/framework/Scoper.java new file mode 100644 index 00000000..b10c3933 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/framework/Scoper.java @@ -0,0 +1,161 @@ +/* Scoper + * + * Created on Jun 6, 2005 + * + * Copyright (C) 2005 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.crawler.framework; + +import java.util.logging.FileHandler; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.crawler.reporting.CrawlerLoggerModule; +import org.archive.crawler.util.LogUtils; +import org.archive.modules.Processor; +import org.archive.modules.deciderules.DecideResult; +import org.archive.modules.deciderules.DecideRule; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.Lifecycle; + +/** + * Base class for Scopers. + * Scopers test CrawlURIs against a scope. + * Scopers allow logging of rejected CrawlURIs. + * @author stack + * @version $Date$, $Revision$ + */ +public abstract class Scoper extends Processor implements Lifecycle { + private static Logger LOGGER = + Logger.getLogger(Scoper.class.getName()); + + FileHandler fileLogger = null; + + /** + * If enabled, override default logger for this class (Default logger writes + * the console). Override logger will instead send all logging to a file + * named for this class in the job log directory. Set the logging level and + * other characteristics of the override logger such as rotation size, + * suffix pattern, etc. in heritrix.properties. This attribute is only + * checked once, on startup of a job. + */ + { + setLogToFile(true); + } + public boolean getLogToFile() { + return (Boolean) kp.get("logToFile"); + } + public void setLogToFile(boolean enabled) { + kp.put("logToFile",enabled); + } + + protected CrawlerLoggerModule loggerModule; + public CrawlerLoggerModule getLoggerModule() { + return this.loggerModule; + } + @Autowired + public void setLoggerModule(CrawlerLoggerModule loggerModule) { + this.loggerModule = loggerModule; + } + + protected DecideRule scope; + public DecideRule getScope() { + return this.scope; + } + @Autowired + public void setDecideRule(DecideRule scope) { + this.scope = scope; + } + + // FIXME: Weirdo log overriding might not work on a per-subclass basis, + // we may need to cut and paste it to the three subclasses, or eliminate + // it in favor of java.util.logging best practice. + // + // Also, eliminating weirdo log overriding would mean we wouldn't need to + // tie into the CrawlController; we'd just need the scope. + + /** + * Constructor. + */ + public Scoper() { + super(); + } + + boolean isRunning = false; + public void start() { + if(isRunning) { + return; + } + if (getLogToFile()) { + // Set up logger for this instance. May have special directives + // since this class can log scope-rejected URLs. + fileLogger = LogUtils.createFileLogger(loggerModule.getPath().getFile(), + this.getClass().getName(), + Logger.getLogger(this.getClass().getName())); + } + isRunning = true; + } + + public boolean isRunning() { + return this.isRunning; + } + public void stop() { + if(fileLogger!=null) { + fileLogger.close(); + } + isRunning = false; + } + + /** + * Schedule the given {@link CrawlURI CrawlURI} with the Frontier. + * @param caUri The CrawlURI to be scheduled. + * @return true if CrawlURI was accepted by crawl scope, false + * otherwise. + */ + protected boolean isInScope(CrawlURI caUri) { + boolean result = false; +// FIXME!: getController().setStateProvider(caUri); + DecideResult dr = scope.decisionFor(caUri); + if (dr == DecideResult.ACCEPT) { + result = true; + if (LOGGER.isLoggable(Level.FINER)) { + LOGGER.finer("Accepted: " + caUri); + } + } else { + outOfScope(caUri); + } + return result; + } + + /** + * Called when a CrawlURI is ruled out of scope. + * Override if you don't want logs as coming from this class. + * @param caUri CrawlURI that is out of scope. + */ + protected void outOfScope(CrawlURI caUri) { + if (!LOGGER.isLoggable(Level.INFO)) { + return; + } + LOGGER.info(caUri.getUURI().toString()); + } + + + +} diff --git a/engine/src/main/java/org/archive/crawler/framework/ToePool.java b/engine/src/main/java/org/archive/crawler/framework/ToePool.java new file mode 100644 index 00000000..38a5e03e --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/framework/ToePool.java @@ -0,0 +1,335 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.framework; + +import java.io.PrintWriter; +import java.util.Iterator; +import java.util.Map; +import java.util.TreeSet; + +import org.archive.crawler.reporting.AlertThreadGroup; +import org.archive.util.ArchiveUtils; +import org.archive.util.Histotable; +import org.archive.util.MultiReporter; + +/** + * A collection of ToeThreads. The class manages the ToeThreads currently + * running. It offers methods for increasing and decreasing their + * number, keeping track of their state and (not necessarily safely) + * killing hung threads. + * + * @author Gordon Mohr + * @author Kristinn Sigurdsson + * + * @see org.archive.crawler.framework.ToeThread + */ +public class ToePool extends ThreadGroup implements MultiReporter { + /** run worker thread slightly lower than usual */ + public static int DEFAULT_TOE_PRIORITY = Thread.NORM_PRIORITY - 1; + + protected CrawlController controller; + protected int nextSerialNumber = 1; + protected int targetSize = 0; + + /** + * Constructor. Creates a pool of ToeThreads. + * + * @param c A reference to the CrawlController for the current crawl. + */ + public ToePool(AlertThreadGroup atg, CrawlController c) { + super(atg, "ToeThreads"); + this.controller = c; + setDaemon(true); + } + + public void cleanup() { + // force all Toes waiting on queues, etc to proceed + Thread[] toes = getToes(); + for(Thread toe : toes) { + if(toe!=null) { + toe.interrupt(); + } + } + this.controller = null; + } + + /** + * @return The number of ToeThreads that are not available (Approximation). + */ + public int getActiveToeCount() { + Thread[] toes = getToes(); + int count = 0; + for (int i = 0; i < toes.length; i++) { + if((toes[i] instanceof ToeThread) && + ((ToeThread)toes[i]).isActive()) { + count++; + } + } + return count; + } + + /** + * @return The number of ToeThreads. This may include killed ToeThreads + * that were not replaced. + */ + public int getToeCount() { + Thread[] toes = getToes(); + int count = 0; + for (int i = 0; i 0) { + // must create threads + for(int i = 1; i <= difference; i++) { + startNewThread(); + } + } else { + // must retire extra threads + int retainedToes = targetSize; + Thread[] toes = this.getToes(); + for (int i = 0; i < toes.length ; i++) { + if(!(toes[i] instanceof ToeThread)) { + continue; + } + retainedToes--; + if (retainedToes>=0) { + continue; // this toe is spared + } + // otherwise: + ToeThread tt = (ToeThread)toes[i]; + tt.retire(); + } + } + } + + /** + * Kills specified thread. Killed thread can be optionally replaced with a + * new thread. + * + *

WARNING: This operation should be used with great care. It may + * destabilize the crawler. + * + * @param threadNumber Thread to kill + * @param replace If true then a new thread will be created to take the + * killed threads place. Otherwise the total number of threads + * will decrease by one. + */ + public void killThread(int threadNumber, boolean replace){ + + Thread[] toes = getToes(); + for (int i = 0; i< toes.length; i++) { + if(! (toes[i] instanceof ToeThread)) { + continue; + } + ToeThread toe = (ToeThread) toes[i]; + if(toe.getSerialNumber()==threadNumber) { + toe.kill(); + } + } + + if(replace){ + // Create a new toe thread to take its place. Replace toe + startNewThread(); + } + } + + private synchronized void startNewThread() { + ToeThread newThread = new ToeThread(this, nextSerialNumber++); + newThread.setPriority(DEFAULT_TOE_PRIORITY); + newThread.start(); + } + + /** + * @return Instance of CrawlController. + */ + public CrawlController getController() { + return controller; + } + + // + // Reporter implementation + // + + public static String STANDARD_REPORT = "standard"; + public static String COMPACT_REPORT = "compact"; + protected static String[] REPORTS = {STANDARD_REPORT,COMPACT_REPORT}; + + public String[] getReports() { + return REPORTS; + } + + public void reportTo(String name, PrintWriter writer) { + if(COMPACT_REPORT.equals(name)) { + compactReportTo(writer); + return; + } + if(name!=null && !STANDARD_REPORT.equals(name)) { + writer.print(name); + writer.print(" not recognized: giving standard report/n"); + } + standardReportTo(writer); + } + + /* (non-Javadoc) + * @see org.archive.util.Reporter#reportTo(java.io.Writer) + */ + protected void standardReportTo(PrintWriter writer) { + writer.print("Toe threads report - " + + ArchiveUtils.get12DigitDate() + "\n"); + writer.print(" Job being crawled: " + + this.controller.getMetadata().getJobName() + "\n"); + writer.print(" Number of toe threads in pool: " + getToeCount() + " (" + + getActiveToeCount() + " active)\n\n"); + + Thread[] toes = this.getToes(); + synchronized (toes) { + for (int i = 0; i < toes.length; i++) { + if (!(toes[i] instanceof ToeThread)) { + continue; + } + ToeThread tt = (ToeThread) toes[i]; + if (tt != null) { + tt.reportTo(writer); + } + } + } + } + + /* (non-Javadoc) + * @see org.archive.util.Reporter#reportTo(java.io.Writer) + */ + protected void compactReportTo(PrintWriter writer) { + writer.print(getToeCount() + " threads (" + getActiveToeCount() + + " active)\n"); + + Thread[] toes = this.getToes(); + boolean legendWritten = false; + // TODO: sort by activity: those with curi the longest at front + synchronized (toes) { + for (int i = 0; i < toes.length; i++) { + if (!(toes[i] instanceof ToeThread)) { + continue; + } + ToeThread tt = (ToeThread) toes[i]; + if (tt != null) { + if(!legendWritten) { + writer.println(tt.singleLineLegend()); + legendWritten = true; + } + tt.singleLineReportTo(writer); + } + } + } + } + + public void singleLineReportTo(PrintWriter w) { + Histotable steps = new Histotable(); + Histotable processors = new Histotable(); + Thread[] toes = getToes(); + for (int i = 0; i < toes.length; i++) { + + if(!(toes[i] instanceof ToeThread)) { + continue; + } + ToeThread tt = (ToeThread)toes[i]; + if(tt!=null) { + steps.tally(tt.getStep()); + processors.tally(tt.getCurrentProcessorName()); + } + } + TreeSet> sortedSteps = steps.getSortedByCounts(); + w.print(getToeCount()); + w.print(" threads: "); + w.print(Histotable.entryString(sortedSteps.first())); + if(sortedSteps.size()>1) { + Iterator> iter = sortedSteps.iterator(); + iter.next(); + w.print(", "); + w.print(Histotable.entryString(iter.next())); + } + if(sortedSteps.size()>2) { + w.print(", etc..."); + } + w.print("; "); + TreeSet> sortedProcessors = processors.getSortedByCounts(); + w.print(Histotable.entryString(sortedProcessors.first())); + if(sortedProcessors.size()>1) { + Iterator> iter = sortedProcessors.iterator(); + iter.next(); + while(iter.hasNext()) { + w.print(", "); + w.print(Histotable.entryString(iter.next())); + } + } + } + + /* (non-Javadoc) + * @see org.archive.util.Reporter#singleLineLegend() + */ + public String singleLineLegend() { + return "total: mostCommonStateTotal secondMostCommonStateTotal"; + } + + public void reportTo(PrintWriter writer) { + reportTo(null,writer); + } + + + public void waitForAll() { + while (true) try { + if (isAllAlive(getToes())) { + return; + } + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new IllegalStateException(e); + } + } + + + private static boolean isAllAlive(Thread[] threads) { + for (Thread t: threads) { + if ((t != null) && (!t.isAlive())) { + return false; + } + } + return true; + } +} diff --git a/engine/src/main/java/org/archive/crawler/framework/ToeThread.java b/engine/src/main/java/org/archive/crawler/framework/ToeThread.java new file mode 100644 index 00000000..cd0d96ba --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/framework/ToeThread.java @@ -0,0 +1,663 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.framework; + +import static org.archive.crawler.datamodel.CoreAttributeConstants.A_RUNTIME_EXCEPTION; +import static org.archive.modules.fetcher.FetchStatusCodes.S_PROCESSING_THREAD_KILLED; +import static org.archive.modules.fetcher.FetchStatusCodes.S_RUNTIME_EXCEPTION; +import static org.archive.modules.fetcher.FetchStatusCodes.S_SERIOUS_ERROR; + +import java.io.PrintWriter; +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; +import java.net.InetAddress; +import java.util.Iterator; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.io.SinkHandlerLogThread; +import org.archive.modules.PostProcessor; +import org.archive.modules.ProcessResult; +import org.archive.modules.Processor; +import org.archive.modules.ProcessorChain; +import org.archive.modules.fetcher.HostResolver; +import org.archive.spring.KeyedProperties; +import org.archive.util.ArchiveUtils; +import org.archive.util.DevUtils; +import org.archive.util.ProgressStatisticsReporter; +import org.archive.util.Recorder; +import org.archive.util.RecorderMarker; +import org.archive.util.MultiReporter; + +import com.sleepycat.util.RuntimeExceptionWrapper; + +/** + * One "worker thread"; asks for CrawlURIs, processes them, + * repeats unless told otherwise. + * + * @author Gordon Mohr + */ +public class ToeThread extends Thread +implements RecorderMarker, MultiReporter, ProgressStatisticsReporter, + HostResolver, SinkHandlerLogThread { + + private static final String STEP_NASCENT = "NASCENT"; + private static final String STEP_ABOUT_TO_GET_URI = "ABOUT_TO_GET_URI"; + private static final String STEP_FINISHED = "FINISHED"; + private static final String STEP_ABOUT_TO_BEGIN_PROCESSOR = + "ABOUT_TO_BEGIN_PROCESSOR"; + private static final String STEP_DONE_WITH_PROCESSORS = + "DONE_WITH_PROCESSORS"; + private static final String STEP_HANDLING_RUNTIME_EXCEPTION = + "HANDLING_RUNTIME_EXCEPTION"; + private static final String STEP_ABOUT_TO_RETURN_URI = + "ABOUT_TO_RETURN_URI"; + private static final String STEP_FINISHING_PROCESS = "FINISHING_PROCESS"; + + private static Logger logger = + Logger.getLogger("org.archive.crawler.framework.ToeThread"); + + private CrawlController controller; + private int serialNumber; + + /** + * Each ToeThead has an instance of HttpRecord that gets used + * over and over by each request. + * + * @see org.archive.util.RecorderMarker + */ + private Recorder httpRecorder = null; + + // private HashMap localProcessors + // = new HashMap(); + private String currentProcessorName = ""; + + private String coreName; + private CrawlURI currentCuri; + private long lastStartTime; + private long lastFinishTime; + + // activity monitoring, debugging, and problem detection + private String step = STEP_NASCENT; + private long atStepSince; + + // default priority; may not be meaningful in recent JVMs + private static final int DEFAULT_PRIORITY = Thread.NORM_PRIORITY-2; + + // indicator that a thread is now surplus based on current desired + // count; it should wrap up cleanly + private volatile boolean shouldRetire = false; + + /** + * Create a ToeThread + * + * @param g ToeThreadGroup + * @param sn serial number + */ + public ToeThread(ToePool g, int sn) { + // TODO: add crawl name? + super(g,"ToeThread #" + sn); + coreName="ToeThread #" + sn + ": "; + controller = g.getController(); + serialNumber = sn; + setPriority(DEFAULT_PRIORITY); + int outBufferSize = controller.getRecorderOutBufferBytes(); + int inBufferSize = controller.getRecorderInBufferBytes(); + httpRecorder = new Recorder(controller.getScratchDir().getFile(), + "tt" + sn + "http", outBufferSize, inBufferSize); + lastFinishTime = System.currentTimeMillis(); + } + + /** (non-Javadoc) + * @see java.lang.Thread#run() + */ + public void run() { + String name = controller.getMetadata().getJobName(); + logger.fine(getName()+" started for order '"+name+"'"); + + try { + while ( true ) { + continueCheck(); + + setStep(STEP_ABOUT_TO_GET_URI); + + CrawlURI curi = controller.getFrontier().next(); + + synchronized(this) { + continueCheck(); + setCurrentCuri(curi); + } + + try { + KeyedProperties.loadOverridesFrom(curi); + processCrawlUri(); + } finally { + KeyedProperties.clearOverridesFrom(curi); + } + + setStep(STEP_ABOUT_TO_RETURN_URI); + continueCheck(); + + synchronized(this) { + controller.getFrontier().finished(currentCuri); + setCurrentCuri(null); + } + + setStep(STEP_FINISHING_PROCESS); + lastFinishTime = System.currentTimeMillis(); + if(shouldRetire) { + break; // from while(true) + } + } + } catch (InterruptedException e) { + // thread interrupted, ok to end + logger.log(Level.FINE,this.getName()+ " ended with Interruption"); + } catch (Exception e) { + // everything else (including interruption) + logger.log(Level.SEVERE,"Fatal exception in "+getName(),e); + } catch (OutOfMemoryError err) { + seriousError(err); + } + + setCurrentCuri(null); + // Do cleanup so that objects can be GC. + this.httpRecorder.closeRecorders(); + this.httpRecorder = null; + + logger.fine(getName()+" finished for order '"+name+"'"); + setStep(STEP_FINISHED); + controller = null; + } + + /** + * Set currentCuri, updating thread name as appropriate + * @param curi + */ + private void setCurrentCuri(CrawlURI curi) { + if(curi==null) { + setName(coreName); + } else { + setName(coreName+curi); + } + currentCuri = curi; + } + + /** + * @param s + */ + private void setStep(String s) { + step=s; + atStepSince = System.currentTimeMillis(); + } + + private void seriousError(Error err) { + // try to prevent timeslicing until we have a chance to deal with OOM + // Note that modern-day JVM priority indifference with native threads + // may make this priority-jumbling pointless + setPriority(DEFAULT_PRIORITY+1); + if (controller!=null) { + // hold all ToeThreads from proceeding to next processor + controller.freeReserveMemory(); + controller.requestCrawlPause(); + if (controller.getFrontier().getFrontierJournal() != null) { + controller.getFrontier().getFrontierJournal().seriousError( + getName() + err.getMessage()); + } + } + + // OutOfMemory etc. + String extraInfo = DevUtils.extraInfo(); + System.err.println("<<<"); + System.err.println(ArchiveUtils.getLog17Date()); + System.err.println(err); + System.err.println(extraInfo); + err.printStackTrace(System.err); + + if (controller!=null) { + PrintWriter pw = new PrintWriter(System.err); + controller.getToePool().compactReportTo(pw); + pw.flush(); + } + System.err.println(">>>"); +// DevUtils.sigquitSelf(); + + String context = "unknown"; + if(currentCuri!=null) { + // update fetch-status, saving original as annotation + currentCuri.getAnnotations().add("err="+err.getClass().getName()); + currentCuri.getAnnotations().add("os"+currentCuri.getFetchStatus()); + currentCuri.setFetchStatus(S_SERIOUS_ERROR); + context = currentCuri.singleLineReport() + " in " + currentProcessorName; + } + String message = "Serious error occured trying " + + "to process '" + context + "'\n" + extraInfo; + logger.log(Level.SEVERE, message.toString(), err); + setPriority(DEFAULT_PRIORITY); + } + + /** + * Perform checks as to whether normal execution should proceed. + * + * If an external interrupt is detected, throw an interrupted exception. + * Used before anything that should not be attempted by a 'zombie' thread + * that the Frontier/Crawl has given up on. + * + * Otherwise, if the controller's memoryGate has been closed, + * hold until it is opened. (Provides a better chance of + * being able to complete some tasks after an OutOfMemoryError.) + * + * @throws InterruptedException + */ + private void continueCheck() throws InterruptedException { + if(Thread.interrupted()) { + throw new InterruptedException("die request detected"); + } + } + + /** + * Pass the CrawlURI to all appropriate processors + * + * @throws InterruptedException + */ + private void processCrawlUri() throws InterruptedException { + assert KeyedProperties.overridesActiveFrom(currentCuri); + + currentCuri.setThreadNumber(this.serialNumber); + lastStartTime = System.currentTimeMillis(); + ProcessorChain localProcessors = + controller.getProcessorChain(); + + currentCuri.setRecorder(httpRecorder); + try { + Iterator iter = localProcessors.iterator(); + Processor curProc = + iter.hasNext() ? iter.next() : null; + while (curProc != null) { + setStep(STEP_ABOUT_TO_BEGIN_PROCESSOR); + currentProcessorName = curProc.getName(); + continueCheck(); + ProcessResult pr = curProc.process(currentCuri); + switch (pr.getProcessStatus()) { + case PROCEED: + curProc = iter.hasNext() ? iter.next() : null; + break; + case STUCK: + controller.requestCrawlPause(); + curProc = null; + break; + case FINISH: + curProc = advanceToPostProcessing(iter); + break; + case JUMP: + curProc = advanceToNamed(iter, pr.getJumpTarget()); + break; + } + } + setStep(STEP_DONE_WITH_PROCESSORS); + currentProcessorName = ""; + } catch (RuntimeExceptionWrapper e) { + // Workaround to get cause from BDB + if(e.getCause() == null) { + e.initCause(e.getCause()); + } + recoverableProblem(e); + } catch (AssertionError ae) { + // This risks leaving crawl in fatally inconsistent state, + // but is often reasonable for per-Processor assertion problems + recoverableProblem(ae); + } catch (RuntimeException e) { + recoverableProblem(e); + } catch (StackOverflowError err) { + recoverableProblem(err); + } catch (Error err) { + // OutOfMemory and any others + seriousError(err); + } + } + + + private Processor advanceToNamed( + Iterator iter, String name) { + while (iter.hasNext()) { + Processor me = iter.next(); + if (me.getName().equals(name)) { + return me; + } + } + return null; + } + + + private Processor advanceToPostProcessing(Iterator iter) { + while (iter.hasNext()) { + Processor me = iter.next(); + if (me instanceof PostProcessor) { + return me; + } + } + return null; + } + + + /** + * Handling for exceptions and errors that are possibly recoverable. + * + * @param e + */ + private void recoverableProblem(Throwable e) { + Object previousStep = step; + setStep(STEP_HANDLING_RUNTIME_EXCEPTION); + e.printStackTrace(System.err); + currentCuri.setFetchStatus(S_RUNTIME_EXCEPTION); + // store exception temporarily for logging + currentCuri.getAnnotations().add("err="+e.getClass().getName()); + currentCuri.getData().put(A_RUNTIME_EXCEPTION, e); + String message = "Problem " + e + + " occured when trying to process '" + + currentCuri.toString() + + "' at step " + previousStep + + " in " + currentProcessorName +"\n"; + logger.log(Level.SEVERE, message.toString(), e); + } + + + /** + * @return Return toe thread serial number. + */ + public int getSerialNumber() { + return this.serialNumber; + } + + /** + * Used to get current threads HttpRecorder instance. + * Implementation of the HttpRecorderMarker interface. + * @return Returns instance of HttpRecorder carried by this thread. + * @see org.archive.util.RecorderMarker#getHttpRecorder() + */ + public Recorder getHttpRecorder() { + return this.httpRecorder; + } + + /** Get the CrawlController acossiated with this thread. + * + * @return Returns the CrawlController. + */ + public CrawlController getController() { + return controller; + } + + /** + * Terminates a thread. + * + *

Calling this method will ensure that the current thread will stop + * processing as soon as possible (note: this may be never). Meant to + * 'short circuit' hung threads. + * + *

Current crawl uri will have its fetch status set accordingly and + * will be immediately returned to the frontier. + * + *

As noted before, this does not ensure that the thread will stop + * running (ever). But once evoked it will not try and communicate with + * other parts of crawler and will terminate as soon as control is + * established. + */ + protected void kill(){ + this.interrupt(); + synchronized(this) { + if (currentCuri!=null) { + currentCuri.setFetchStatus(S_PROCESSING_THREAD_KILLED); + controller.getFrontier().finished(currentCuri); + } + } + } + + /** + * @return Current step (For debugging/reporting, give abstract step + * where this thread is). + */ + public Object getStep() { + return step; + } + + /** + * Is this thread validly processing a URI, not paused, waiting for + * a URI, or interrupted? + * @return whether thread is actively processing a URI + */ + public boolean isActive() { + // if alive and not waiting in/for frontier.next(), we're 'active' + return this.isAlive() && (currentCuri != null) && !isInterrupted(); + } + + /** + * Request that this thread retire (exit cleanly) at the earliest + * opportunity. + */ + public void retire() { + shouldRetire = true; + } + + /** + * Whether this thread should cleanly retire at the earliest + * opportunity. + * + * @return True if should retire. + */ + public boolean shouldRetire() { + return shouldRetire; + } + + // + // Reporter implementation + // + + /** + * Compiles and returns a report on its status. + * @param name Report name. + * @param pw Where to print. + */ + public void reportTo(String name, PrintWriter pw) { + // name is ignored for now: only one kind of report + + pw.print("["); + pw.println(getName()); + + // Make a local copy of the currentCuri reference in case it gets + // nulled while we're using it. We're doing this because + // alternative is synchronizing and we don't want to do this -- + // it causes hang ups as controller waits on a lock for this thread, + // something it gets easily enough on old threading model but something + // it can wait interminably for on NPTL threading model. + // See [ 994946 ] Pause/Terminate ignored on 2.6 kernel 1.5 JVM. + CrawlURI c = currentCuri; + if(c != null) { + pw.print(" "); + c.singleLineReportTo(pw); + pw.print(" "); + pw.print(c.getFetchAttempts()); + pw.print(" attempts"); + pw.println(); + pw.print(" "); + pw.print("in processor: "); + pw.print(currentProcessorName); + } else { + pw.print(" -no CrawlURI- "); + } + pw.println(); + + long now = System.currentTimeMillis(); + long time = 0; + + pw.print(" "); + if(lastFinishTime > lastStartTime) { + // That means we finished something after we last started something + // or in other words we are not working on anything. + pw.print("WAITING for "); + time = now - lastFinishTime; + } else if(lastStartTime > 0) { + // We are working on something + pw.print("ACTIVE for "); + time = now-lastStartTime; + } + pw.print(ArchiveUtils.formatMillisecondsToConventional(time)); + pw.println(); + + pw.print(" "); + pw.print("step: "); + pw.print(step); + pw.print(" for "); + pw.print(ArchiveUtils.formatMillisecondsToConventional(System.currentTimeMillis()-atStepSince)); + pw.println(); + + reportThread(this, pw); + pw.print("]"); + pw.println(); + + pw.flush(); + } + + /** + * @param t Thread + * @param pw PrintWriter + */ + static public void reportThread(Thread t, PrintWriter pw) { + ThreadMXBean tmxb = ManagementFactory.getThreadMXBean(); + ThreadInfo info = tmxb.getThreadInfo(t.getId()); + pw.print("Java Thread State: "); + pw.println(info.getThreadState()); + pw.print("Blocked/Waiting On: "); + if (info.getLockOwnerId() >= 0) { + pw.print(info.getLockName()); + pw.print(" which is owned by "); + pw.print(info.getLockOwnerName()); + pw.print("("); + pw.print(info.getLockOwnerId()); + pw.println(")"); + } else { + pw.println("NONE"); + } + + StackTraceElement[] ste = t.getStackTrace(); + for(int i=0;i lastStartTime) { + // That means we finished something after we last started something + // or in other words we are not working on anything. + w.print("WAITING for "); + time = now - lastFinishTime; + } else if(lastStartTime > 0) { + // We are working on something + w.print("ACTIVE for "); + time = now-lastStartTime; + } + w.print(ArchiveUtils.formatMillisecondsToConventional(time)); + w.print(" at "); + w.print(step); + w.print(" for "); + w.print(ArchiveUtils.formatMillisecondsToConventional(now-atStepSince)); + w.print("\n"); + w.flush(); + } + + /* (non-Javadoc) + * @see org.archive.util.Reporter#singleLineLegend() + */ + public String singleLineLegend() { + return "#serialNumber processorName currentUri (fetchAttempts) threadState threadStep"; + } + + /* (non-Javadoc) + * @see org.archive.util.Reporter#getReports() + */ + public String[] getReports() { + // for now none but the default + return new String[] {}; + } + + public void reportTo(PrintWriter writer) { + reportTo(null, writer); + } + + /* (non-Javadoc) + * @see org.archive.util.Reporter#singleLineReport() + */ + public String singleLineReport() { + return ArchiveUtils.singleLineReport(this); + } + + public void progressStatisticsLine(PrintWriter writer) { + writer.print(getController().getStatisticsTracker() + .getSnapshot().getProgressStatisticsLine()); + writer.print("\n"); + } + + public void progressStatisticsLegend(PrintWriter writer) { + writer.print(getController().getStatisticsTracker() + .progressStatisticsLegend()); + writer.print("\n"); + } + + public String getCurrentProcessorName() { + return currentProcessorName; + } + + + public InetAddress resolve(String host) { + return controller.getServerCache().getHostFor(host).getIP(); + } +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java new file mode 100644 index 00000000..30f68033 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java @@ -0,0 +1,1645 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.frontier; + +import static org.archive.crawler.datamodel.CoreAttributeConstants.A_FETCH_COMPLETED_TIME; +import static org.archive.crawler.datamodel.CoreAttributeConstants.A_NONFATAL_ERRORS; +import static org.archive.modules.fetcher.FetchStatusCodes.S_BLOCKED_BY_CUSTOM_PROCESSOR; +import static org.archive.modules.fetcher.FetchStatusCodes.S_BLOCKED_BY_USER; +import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED; +import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST; +import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED; +import static org.archive.modules.fetcher.FetchStatusCodes.S_DELETED_BY_USER; +import static org.archive.modules.fetcher.FetchStatusCodes.S_DOMAIN_UNRESOLVABLE; +import static org.archive.modules.fetcher.FetchStatusCodes.S_OUT_OF_SCOPE; +import static org.archive.modules.fetcher.FetchStatusCodes.S_ROBOTS_PRECLUDED; +import static org.archive.modules.fetcher.FetchStatusCodes.S_TOO_MANY_EMBED_HOPS; +import static org.archive.modules.fetcher.FetchStatusCodes.S_TOO_MANY_LINK_HOPS; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.PrintWriter; +import java.io.Serializable; +import java.io.StringWriter; +import java.io.Writer; +import java.util.Collection; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.httpclient.HttpStatus; +import org.apache.commons.httpclient.URIException; +import org.archive.checkpointing.CheckpointRecovery; +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.crawler.datamodel.SchedulingConstants; +import org.archive.crawler.event.CrawlStateEvent; +import org.archive.crawler.framework.CrawlController; +import org.archive.crawler.framework.Frontier; +import org.archive.crawler.reporting.CrawlerLoggerModule; +import org.archive.crawler.spring.SheetOverlaysManager; +import org.archive.modules.ModuleAttributeConstants; +import org.archive.modules.ProcessorURI; +import org.archive.modules.canonicalize.UriCanonicalizationPolicy; +import org.archive.modules.deciderules.DecideRule; +import org.archive.modules.extractor.ExtractorParameters; +import org.archive.modules.fetcher.UserAgentProvider; +import org.archive.modules.fetcher.FetchStats.Stage; +import org.archive.modules.net.CrawlHost; +import org.archive.modules.net.CrawlServer; +import org.archive.modules.net.RobotsExclusionPolicy; +import org.archive.modules.net.ServerCache; +import org.archive.modules.net.ServerCacheUtil; +import org.archive.modules.seeds.SeedListener; +import org.archive.modules.seeds.SeedModule; +import org.archive.net.UURI; +import org.archive.spring.ConfigPath; +import org.archive.spring.HasKeyedProperties; +import org.archive.spring.KeyedProperties; +import org.archive.util.ArchiveUtils; +import org.archive.util.iterator.LineReadingIterator; +import org.archive.util.iterator.RegexpLineIterator; +import org.json.JSONException; +import org.json.JSONObject; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationEvent; +import org.springframework.context.Lifecycle; + +/** + * Shared facilities for Frontier implementations. + * + * @author gojomo + */ +public abstract class AbstractFrontier + implements Frontier, + Serializable, + Lifecycle, // InitializingBean, + SeedListener, + HasKeyedProperties, + ExtractorParameters { + private static final long serialVersionUID = 555881755284996860L; + private static final Logger logger = Logger + .getLogger(AbstractFrontier.class.getName()); + + KeyedProperties kp = new KeyedProperties(); + public KeyedProperties getKeyedProperties() { + return kp; + } + + + public boolean isRunning() { + return managerThread!=null && managerThread.isAlive(); + } + + public void stop() { + terminate(); + } + + + protected CrawlController controller; + public CrawlController getCrawlController() { + return this.controller; + } + @Autowired + public void setCrawlController(CrawlController controller) { + this.controller = controller; + } + + protected SheetOverlaysManager sheetOverlaysManager; + public SheetOverlaysManager getSheetOverlaysManager() { + return sheetOverlaysManager; + } + @Autowired + public void setSheetOverlaysManager(SheetOverlaysManager sheetOverlaysManager) { + this.sheetOverlaysManager = sheetOverlaysManager; + } + + protected CrawlerLoggerModule loggerModule; + public CrawlerLoggerModule getLoggerModule() { + return this.loggerModule; + } + @Autowired + public void setLoggerModule(CrawlerLoggerModule loggerModule) { + this.loggerModule = loggerModule; + } + + protected SeedModule seeds; + public SeedModule getSeeds() { + return this.seeds; + } + @Autowired + public void setSeeds(SeedModule seeds) { + this.seeds = seeds; + } + + protected ServerCache serverCache; + public ServerCache getServerCache() { + return this.serverCache; + } + @Autowired + public void setServerCache(ServerCache serverCache) { + this.serverCache = serverCache; + } + + /** ordinal numbers to assign to created CrawlURIs */ + protected AtomicLong nextOrdinal = new AtomicLong(1); + + protected DecideRule scope; + public DecideRule getScope() { + return this.scope; + } + @Autowired + public void setScope(DecideRule scope) { + this.scope = scope; + } + + protected ConfigPath recoveryDir = new ConfigPath("recovery subdirectory","logs"); + public ConfigPath getRecoveryDir() { + return recoveryDir; + } + public void setRecoveryDir(ConfigPath recoveryDir) { + this.recoveryDir = recoveryDir; + } + + /** + * How many multiples of last fetch elapsed time to wait before recontacting + * same server. + */ + { + setDelayFactor(5.0f); + } + public float getDelayFactor() { + return (Float) kp.get("delayFactor"); + } + public void setDelayFactor(float factor) { + kp.put("delayFactor",factor); + } + + /** + * always wait this long after one completion before recontacting same + * server, regardless of multiple + */ + { + setMinDelayMs(3000); + } + public int getMinDelayMs() { + return (Integer) kp.get("minDelayMs"); + } + public void setMinDelayMs(int minDelay) { + kp.put("minDelayMs",minDelay); + } + + /** + * Whether to respect a 'Crawl-Delay' (in seconds) given in a site's + * robots.txt + */ + { + setRespectCrawlDelay(true); + } + public boolean getRespectCrawlDelay() { + return (Boolean) kp.get("respectCrawlDelay"); + } + public void setRespectCrawlDelay(boolean respect) { + kp.put("respectCrawlDelay",respect); + } + + /** never wait more than this long, regardless of multiple */ + { + setMaxDelayMs(30000); + } + public int getMaxDelayMs() { + return (Integer) kp.get("maxDelayMs"); + } + public void setMaxDelayMs(int maxDelay) { + kp.put("maxDelayMs",maxDelay); + } + + /** number of hops of embeds (ERX) to bump to front of host queue */ + { + setPreferenceEmbedHops(1); + } + public int getPreferenceEmbedHops() { + return (Integer) kp.get("preferenceEmbedHops"); + } + public void setPreferenceEmbedHops(int pref) { + kp.put("preferenceEmbedHops",pref); + } + + /** maximum per-host bandwidth usage */ + { + setMaxPerHostBandwidthUsageKbSec(0); + } + public int getMaxPerHostBandwidthUsageKbSec() { + return (Integer) kp.get("maxPerHostBandwidthUsageKbSec"); + } + public void setMaxPerHostBandwidthUsageKbSec(int max) { + kp.put("maxPerHostBandwidthUsageKbSec",max); + } + + /** maximum overall bandwidth usage */ + { + setTotalBandwidthUsageKbSec(0); + } + public int getTotalBandwidthUsageKbSec() { + return (Integer) kp.get("totalBandwidthUsageKbSec"); + } + public void setTotalBandwidthUsageKbSec(int total) { + kp.put("totalBandwidthUsageKbSec",total); + } + + /** for retryable problems, seconds to wait before a retry */ + { + setRetryDelaySeconds(900); + } + public int getRetryDelaySeconds() { + return (Integer) kp.get("retryDelaySeconds"); + } + public void setRetryDelaySeconds(int delay) { + kp.put("retryDelaySeconds",delay); + } + + /** maximum times to emit a CrawlURI without final disposition */ + { + setMaxRetries(30); + } + public int getMaxRetries() { + return (Integer) kp.get("maxRetries"); + } + public void setMaxRetries(int maxRetries) { + kp.put("maxRetries",maxRetries); + } + + /** size of the 'outbound' mediation queue between manager thread + * and toethreads */ + int outboundQueueCapacity = 50; + public int getOutboundQueueCapacity() { + return this.outboundQueueCapacity; + } + public void setOutboundQueueCapacity(int capacity) { + this.outboundQueueCapacity = capacity; + } + + /** size of the inbound queue as multiple of the outbound queue */ + int inboundQueueMultiple = 3; + public int getInboundQueueMultiple() { + return this.inboundQueueMultiple; + } + public void setInboundQueueMultiple(int multiple) { + this.inboundQueueMultiple = multiple; + } + + + /** queue assignment to force onto CrawlURIs; intended to be overridden */ + { + setForceQueueAssignment(""); + } + public String getForceQueueAssignment() { + return (String) kp.get("forceQueueAssignment"); + } + public void setForceQueueAssignment(String forceQueueAssignment) { + kp.put("forceQueueAssignment",forceQueueAssignment); + } + + // word chars, dash, period, comma, colon +// protected final static String ACCEPTABLE_FORCE_QUEUE = "[-\\w\\.,:]*"; + + /** + * Whether to tag seeds with their own URI as a heritable 'source' String, + * which will be carried-forward to all URIs discovered on paths originating + * from that seed. When present, such source tags appear in the + * second-to-last crawl.log field. + */ + { + setSourceTagSeeds(false); + } + public boolean getSourceTagSeeds() { + return (Boolean) kp.get("sourceTagSeeds"); + } + public void setSourceTagSeeds(boolean sourceTagSeeds) { + kp.put("sourceTagSeeds",sourceTagSeeds); + } + + /** + * Recover log on or off attribute. + */ + { + setRecoveryLogEnabled(true); + } + public boolean getRecoveryLogEnabled() { + return (Boolean) kp.get("recoveryLogEnabled"); + } + public void setRecoveryLogEnabled(boolean enabled) { + kp.put("recoveryLogEnabled",enabled); + } + + { + setMaxOutlinks(6000); + } + public int getMaxOutlinks() { + return (Integer) kp.get("maxOutlinks"); + } + public void setMaxOutlinks(int max) { + kp.put("maxOutlinks", max); + } + + // top-level stats + /** total URIs queued to be visited */ + protected AtomicLong queuedUriCount = new AtomicLong(0); + + protected AtomicLong succeededFetchCount = new AtomicLong(0); + + protected AtomicLong failedFetchCount = new AtomicLong(0); + + /** URIs that are disregarded (for example because of robot.txt rules */ + protected AtomicLong disregardedUriCount = new AtomicLong(0); + + /** + * Used when bandwidth constraint are used. + */ + protected long totalProcessedBytes = 0; + + protected long processedBytesAfterLastEmittedURI = 0; + + protected int lastMaxBandwidthKB = 0; + + /** + * Crawl replay logger. + * + * Currently captures Frontier/URI transitions. + * Can be null if user chose not to run a recovery.log. + */ + private transient FrontierJournal recover = null; + + /** file collecting report of ignored seed-file entries (if any) */ + public static final String IGNORED_SEEDS_FILENAME = "seeds.ignored"; + + /** + * Ordered list of url canonicalization rules. Rules are applied in the + * order listed from top to bottom. + */ + + @SuppressWarnings("unchecked") + public UriCanonicalizationPolicy getUriCanonicalizationPolicy() { + return (UriCanonicalizationPolicy) kp.get("uriCanonicalizationRules"); + } + @Autowired + public void setCanonicalizationPolicy(UriCanonicalizationPolicy policy) { + kp.put("uriCanonicalizationRules",policy); + } + + /** + * Defines how to assign URIs to queues. Can assign by host, by ip, + * by SURT-ordered authority, by SURT-ordered authority truncated to + * a topmost-assignable domain, and into one of a fixed set of buckets + * (1k). + */ + { + setQueueAssignmentPolicy(new SurtAuthorityQueueAssignmentPolicy()); + } + public QueueAssignmentPolicy getQueueAssignmentPolicy() { + return (QueueAssignmentPolicy) kp.get("queueAssignmentPolicy"); + } + @Autowired(required=false) + public void setQueueAssignmentPolicy(QueueAssignmentPolicy policy) { + kp.put("queueAssignmentPolicy",policy); + } + + /** + * Auto-discovered module providing configured (or overridden) + * User-Agent value; now necessary in frontier because User-Agent + * may affect politeness delays via robots.txt Crawl-Delay. + */ + public UserAgentProvider getUserAgentProvider() { + return (UserAgentProvider) kp.get("userAgentProvider"); + } + @Autowired + public void setUserAgentProvider(UserAgentProvider provider) { + kp.put("userAgentProvider",provider); + } + + + /** + * @param name Name of this frontier. + * @param description Description for this frontier. + */ + public AbstractFrontier() { + + } + + /** reusable no-op inbound event, to force reeval of state/eligible URIs */ + transient protected InEvent NOOP = new InEvent() { public void process() {} }; + + /** inbound updates: URIs to be scheduled, finished; requested state changes */ + transient protected ArrayBlockingQueue inbound; + /** outbound URIs */ + transient protected ArrayBlockingQueue outbound; + + /** Capacity of the inbound queue. */ + private int inboundCapacity; + + /** Capacity of the outbound queue. */ + private int outboundCapacity; + + /** + * lock to allow holding all worker ToeThreads from taking URIs already + * on the outbound queue; they acquire read permission before take()ing; + * frontier can acquire write permission to hold threads */ + protected ReentrantReadWriteLock outboundLock = + new ReentrantReadWriteLock(true); + + + /** + * Distinguished frontier manager thread which handles all juggling + * of URI queues and queues/maps of queues for proper ordering/delay of + * URI processing. + */ + transient Thread managerThread; + + /** last Frontier.State reached; used to suppress duplicate notifications */ + State lastReachedState = null; + /** Frontier.state that manager thread should seek to reach */ + State targetState = State.PAUSE; + + /** + * Start the manager thread. Subclasses should invoke this in a custom + * readObject() method so that the manager thread begins after checkpoint + * recovery. + */ + protected void startManagerThread() { + managerThread = new Thread(this+".managerThread") { + public void run() { + AbstractFrontier.this.managementTasks(); + } + }; + managerThread.start(); + } + + public void start() { + if(isRunning()) { + return; + } + seeds.addSeedListener(this); + + if (getRecoveryLogEnabled()) try { + initJournal(loggerModule.getPath().getFile().getAbsolutePath()); + } catch (IOException e) { + throw new IllegalStateException(e); + } + + this.outboundCapacity = getOutboundQueueCapacity(); + this.inboundCapacity = outboundCapacity * + getInboundQueueMultiple(); + outbound = new ArrayBlockingQueue(outboundCapacity, true); + inbound = new ArrayBlockingQueue(inboundCapacity, true); + pause(); + startManagerThread(); + } + + /** + * Main loop of frontier's managerThread. Only exits when State.FINISH + * is requested (perhaps automatically at URI exhaustion) and reached. + * + * General strategy is to try to fill outbound queue, then process an + * item from inbound queue, and repeat. A HOLD (to be implemented) or + * PAUSE puts frontier into a stable state that won't be changed + * asynchronously by worker thread activity. + */ + protected void managementTasks() { + assert Thread.currentThread() == managerThread; + try { + loop: while (true) { + try { + switch (targetState) { + case RUN: + // enable outbound takes if previously locked + while(outboundLock.isWriteLockedByCurrentThread()) { + outboundLock.writeLock().unlock(); + } + reachedState(State.RUN); + // fill to-do 'on-deck' queue + fillOutbound(); + // process discovered and finished URIs + drainInbound(); + if(isEmpty()) { + // pause when frontier exhausted; controller will + // determine if this means to finish or not + targetState = State.PAUSE; + } + break; + case HOLD: + // TODO; for now treat same as PAUSE + case PAUSE: + // pausing + // prevent all outbound takes + outboundLock.writeLock().lock(); + // process all inbound + while (targetState == State.PAUSE) { + if (outbound.size() == getInProcessCount()) { + // if all 'in-process' URIs are actually + // waiting in outbound, we are at PAUSE + reachedState(State.PAUSE); + } + // continue to process discovered and finished URIs + inbound.take().process(); + } + break; + case FINISH: + // prevent all outbound takes + outboundLock.writeLock().lock(); + // process all inbound + while (outbound.size() != getInProcessCount()) { + // continue to process discovered and finished URIs + inbound.take().process(); + } + // TODO: more cleanup? + reachedState(State.FINISH); + break loop; + } + } catch (RuntimeException e) { + // log, try to pause, continue + logger.log(Level.SEVERE,"",e); + if(targetState!=State.PAUSE) { + requestState(State.PAUSE); + } + } + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + // try to leave in safely restartable state: + targetState = State.PAUSE; + while(outboundLock.isWriteLockedByCurrentThread()) { + outboundLock.writeLock().unlock(); + } + //TODO: ensure all other structures are cleanly reset on restart + + logger.log(Level.FINE,"ending frontier mgr thread"); + } + + + /** + * Fill the outbound queue with eligible CrawlURIs, to capacity + * or as much as possible. + * + * @throws InterruptedException + */ + protected void fillOutbound() throws InterruptedException { + while (outbound.remainingCapacity() > 0) { + CrawlURI crawlable = findEligibleURI(); + if (crawlable != null) { + outbound.put(crawlable); + } else { + break; + } + } + } + + /** + * Drain the inbound queue of update events, or at the very least + * wait until some additional delayed-queue URI becomes available. + * + * @throws InterruptedException + */ + protected void drainInbound() throws InterruptedException { + int batch = inbound.size(); + for(int i = 0; i < batch; i++) { + inbound.take().process(); + } + if(batch==0) { + // always do at least one timed try + InEvent toProcess = inbound.poll(getMaxInWait(), + TimeUnit.MILLISECONDS); + if (toProcess != null) { + toProcess.process(); + } + } + } + + /** + * The given state has been reached; if it is a new state, generate + * a notification to the CrawlController. + * + * TODO: evaluate making this a generic notification others can sign up for + */ + protected void reachedState(State justReached) { + if(justReached != lastReachedState) { + controller.noteFrontierState(justReached); + lastReachedState = justReached; + } + } + + /* (non-Javadoc) + * @see org.archive.crawler.framework.Frontier#next() + */ + public CrawlURI next() throws InterruptedException { + // perhaps hold without taking ready outbound items + outboundLock.readLock().lockInterruptibly(); + outboundLock.readLock().unlock(); + + + CrawlURI retval = outbound.take(); +// // TODO: consider if following necessary for maintaining throughput +// if(outbound.size()<=1) { +// doOrEnqueue(NOOP); +// } + return retval; + } + + /** + * Find a CrawlURI eligible to be put on the outbound queue for + * processing. If none, return null. + * @return the eligible URI, or null + */ + abstract protected CrawlURI findEligibleURI(); + + + /** + * Schedule the given CrawlURI regardless of its already-seen status. Only + * to be called inside the managerThread, as by an InEvent. + * + * @param caUri CrawlURI to schedule + */ + abstract protected void processScheduleAlways(CrawlURI caUri); + + /** + * Schedule the given CrawlURI if not already-seen. Only + * to be called inside the managerThread, as by an InEvent. + * + * @param caUri CrawlURI to schedule + */ + abstract protected void processScheduleIfUnique(CrawlURI caUri); + + /** + * Handle the given CrawlURI as having finished a worker ToeThread + * processing attempt. May result in the URI being rescheduled or + * logged as successful or failed. Only to be called inside the + * managerThread, as by an InEvent. + * + * @param caUri CrawlURI to finish + */ + abstract protected void processFinish(CrawlURI caUri); + + /** + * The number of CrawlURIs 'in process' (passed to the outbound + * queue and not yet finished by returning through the inbound + * queue.) + * + * @return number of in-process CrawlURIs + */ + abstract protected int getInProcessCount(); + + + /** + * Maximum amount of time to wait for an inbound update event before + * giving up and rechecking on the ability to further fill the outbound + * queue. If any queues are waiting out politeness/retry delays ('snoozed'), + * the maximum wait should be no longer than the shortest sch delay. + * @return maximum time to wait, in milliseconds + */ + abstract protected long getMaxInWait(); + + /** + * Arrange for the given CrawlURI to be visited, if it is not + * already scheduled/completed. + * + * @see org.archive.crawler.framework.Frontier#schedule(org.archive.crawler.datamodel.CrawlURI) + */ + public void schedule(CrawlURI curi) { + applyOverridesTo(curi); + enqueueOrDo(new ScheduleIfUnique(curi)); + } + + /** + * Accept the given CrawlURI for scheduling, as it has + * passed the alreadyIncluded filter. + * + * Choose a per-classKey queue and enqueue it. If this + * item has made an unready queue ready, place that + * queue on the readyClassQueues queue. + * @param caUri CrawlURI. + */ + public void receive(CrawlURI curi) { + applyOverridesTo(curi); + // prefer doing asap if already in manager thread + doOrEnqueue(new ScheduleAlways(curi)); + } + + /** + * Note that the previously emitted CrawlURI has completed + * its processing (for now). + * + * The CrawlURI may be scheduled to retry, if appropriate, + * and other related URIs may become eligible for release + * via the next next() call, as a result of finished(). + * + * (non-Javadoc) + * @see org.archive.crawler.framework.Frontier#finished(org.archive.crawler.datamodel.CrawlURI) + */ + public void finished(CrawlURI curi) { + enqueueOrDo(new Finish(curi)); + } + + private void initJournal(String logsDisk) throws IOException { + if (logsDisk != null) { + String logsPath = logsDisk + File.separatorChar; + this.recover = new FrontierJournal(logsPath, + FrontierJournal.LOGNAME_RECOVER); + } + } + + public void run() { + requestState(State.RUN); + } + + /* (non-Javadoc) + * @see org.archive.crawler.framework.Frontier#requestState(org.archive.crawler.framework.Frontier.State) + */ + public void requestState(State target) { + enqueueOrDo(new SetTargetState(target)); + } + + /** + * Actually set a new target Frontier.State. Should only be called in + * managerThread, as by an InEvent. + */ + protected void processSetTargetState(State target) { + assert Thread.currentThread() == managerThread; + targetState = target; + } + + public void pause() { + requestState(State.PAUSE); + } + + public void unpause() { + requestState(State.RUN); + } + + + synchronized public void terminate() { + requestState(State.FINISH); + // TODO: move this recover-cleanup to manager thread? + if (this.recover != null) { + this.recover.close(); + this.recover = null; + } + } + + /** + * Report CrawlURI to each of the three 'substats' accumulators + * (group/queue, server, host) for a given stage. + * + * @param curi + * @param stage + */ + protected void tally(CrawlURI curi, Stage stage) { + // Tally per-server, per-host, per-frontier-class running totals + CrawlServer server = ServerCacheUtil.getServerFor( + getServerCache(),curi.getUURI()); + if (server != null) { + server.getSubstats().tally(curi, stage); + } + CrawlHost host = ServerCacheUtil.getHostFor( + getServerCache(),curi.getUURI()); + if (host != null) { + host.getSubstats().tally(curi, stage); + } + FrontierGroup group = getGroup(curi); + group.tally(curi, stage); + } + + protected void doJournalFinishedSuccess(CrawlURI c) { + tally(c,Stage.SUCCEEDED); + if (this.recover != null) { + this.recover.finishedSuccess(c); + } + } + + protected void doJournalAdded(CrawlURI c) { + tally(c,Stage.SCHEDULED); + if (this.recover != null) { + this.recover.added(c); + } + } + + protected void doJournalRelocated(CrawlURI c) { + tally(c,Stage.RELOCATED); + if (this.recover != null) { + // TODO: log dequeue from original location somehow + // this.recover.relocated(c); + } + } + + protected void doJournalRescheduled(CrawlURI c) { + tally(c,Stage.RETRIED); + if (this.recover != null) { + this.recover.rescheduled(c); + } + } + + protected void doJournalFinishedFailure(CrawlURI c) { + tally(c,Stage.FAILED); + if (this.recover != null) { + this.recover.finishedFailure(c); + } + } + + protected void doJournalDisregarded(CrawlURI c) { + tally(c, Stage.DISREGARDED); + if (this.recover != null) { + this.recover.finishedDisregard(c); + } + } + + protected void doJournalEmitted(CrawlURI c) { + if (this.recover != null) { + this.recover.emitted(c); + } + } + + /** + * Frontier is empty only if all queues are empty and no URIs are in-process + * + * @return True if queues are empty. + */ + public boolean isEmpty() { + return queuedUriCount.get() == 0; + } + + /** + * Increment the running count of queued URIs. + */ + protected void incrementQueuedUriCount() { + queuedUriCount.incrementAndGet(); + } + + /** + * Increment the running count of queued URIs. Synchronized because + * operations on longs are not atomic. + * + * @param increment + * amount to increment the queued count + */ + protected void incrementQueuedUriCount(long increment) { + queuedUriCount.addAndGet(increment); + } + + /** + * Note that a number of queued Uris have been deleted. + * + * @param numberOfDeletes + */ + protected void decrementQueuedCount(long numberOfDeletes) { + queuedUriCount.addAndGet(-numberOfDeletes); + } + + /** + * (non-Javadoc) + * + * @see org.archive.crawler.framework.Frontier#queuedUriCount() + */ + public long queuedUriCount() { + return queuedUriCount.get(); + } + + /** + * (non-Javadoc) + * + * @see org.archive.crawler.framework.Frontier#finishedUriCount() + */ + public long finishedUriCount() { + return succeededFetchCount.get() + failedFetchCount.get() + disregardedUriCount.get(); + } + + /** + * Increment the running count of successfully fetched URIs. + */ + protected void incrementSucceededFetchCount() { + succeededFetchCount.incrementAndGet(); + } + + /** + * (non-Javadoc) + * + * @see org.archive.crawler.framework.Frontier#succeededFetchCount() + */ + public long succeededFetchCount() { + return succeededFetchCount.get(); + } + + /** + * Increment the running count of failed URIs. + */ + protected void incrementFailedFetchCount() { + failedFetchCount.incrementAndGet(); + } + + /** + * (non-Javadoc) + * + * @see org.archive.crawler.framework.Frontier#failedFetchCount() + */ + public long failedFetchCount() { + return failedFetchCount.get(); + } + + /** + * Increment the running count of disregarded URIs. + */ + protected void incrementDisregardedUriCount() { + disregardedUriCount.incrementAndGet(); + } + + public long disregardedUriCount() { + return disregardedUriCount.get(); + } + + /** @deprecated misnomer; use StatisticsTracker figures instead */ + public long totalBytesWritten() { + return totalProcessedBytes; + } + + /** + * Load up the seeds. + * + * This method is called on initialize and inside in the crawlcontroller + * when it wants to force reloading of configuration. + */ + public void loadSeeds() { + logger.info("beginning"); + // Get the seeds to refresh. + Writer ignoredWriter = new StringWriter(); + Iterator iter = getSeeds().seedsIterator(ignoredWriter); + int count = 0; + while (iter.hasNext()) { + UURI u = (UURI)iter.next(); + CrawlURI caUri = new CrawlURI(u); + caUri.setSeed(true); + caUri.setSchedulingDirective(SchedulingConstants.MEDIUM); + if (getSourceTagSeeds()) { + caUri.setSourceTag(caUri.toString()); + } + schedule(caUri); + count++; + if(count%1000==0) { + logger.info(count+" seeds"); + } + } + // save ignored items (if any) where they can be consulted later + saveIgnoredItems(ignoredWriter.toString(), getRecoveryDir().getFile()); + logger.info("finished"); + } + + + public void seedsRefreshed() { + loadSeeds(); + } + + public void addedSeed(ProcessorURI puri) { + // TODO: schedule? + } + + /** + * Dump ignored seed items (if any) to disk; delete file otherwise. + * Static to allow non-derived sibling classes (frontiers not yet + * subclassed here) to reuse. + * + * @param ignoredItems + * @param dir + */ + public static void saveIgnoredItems(String ignoredItems, File dir) { + File ignoredFile = new File(dir, IGNORED_SEEDS_FILENAME); + if(ignoredItems==null | ignoredItems.length()>0) { + try { + BufferedWriter bw = new BufferedWriter(new FileWriter(ignoredFile)); + bw.write(ignoredItems); + bw.close(); + } catch (IOException e) { + // TODO make an alert? + e.printStackTrace(); + } + } else { + // delete any older file (if any) + ignoredFile.delete(); + } + } + + protected void prepForFrontier(CrawlURI curi) { + if (curi.getOrdinal() == 0) { + curi.setOrdinal(nextOrdinal.getAndIncrement()); + } + curi.setClassKey(getClassKey(curi)); + } + + /** + * Perform any special handling of the CrawlURI, such as promoting its URI + * to seed-status, or preferencing it because it is an embed. + * + * @param curi + */ + protected void applySpecialHandling(CrawlURI curi) { + if (curi.isSeed() && curi.getVia() != null + && curi.flattenVia().length() > 0) { + // The only way a seed can have a non-empty via is if it is the + // result of a seed redirect. Add it to the seeds list. + // + // This is a feature. This is handling for case where a seed + // gets immediately redirected to another page. What we're doing is + // treating the immediate redirect target as a seed. + getSeeds().addSeed(curi); + // And it needs rapid scheduling. + if (curi.getSchedulingDirective() == SchedulingConstants.NORMAL) + curi.setSchedulingDirective(SchedulingConstants.MEDIUM); + } + + // optionally preferencing embeds up to MEDIUM + int prefHops = getPreferenceEmbedHops(); + if (prefHops > 0) { + int embedHops = curi.getTransHops(); + if (embedHops > 0 && embedHops <= prefHops + && curi.getSchedulingDirective() == SchedulingConstants.NORMAL) { + // number of embed hops falls within the preferenced range, and + // uri is not already MEDIUM -- so promote it + curi.setSchedulingDirective(SchedulingConstants.MEDIUM); + } + } + } + + /** + * Perform fixups on a CrawlURI about to be returned via next(). + * + * @param curi + * CrawlURI about to be returned by next() + * @param q + * the queue from which the CrawlURI came + */ + protected void noteAboutToEmit(CrawlURI curi, WorkQueue q) { + curi.setHolder(q); + // if (curi.getServer() == null) { + // // TODO: perhaps short-circuit the emit here, + // // because URI will be rejected as unfetchable + // } + doJournalEmitted(curi); + } + + /** + * Return a suitable value to wait before retrying the given URI. + * + * @param curi + * CrawlURI to be retried + * @return millisecond delay before retry + */ + protected long retryDelayFor(CrawlURI curi) { + int status = curi.getFetchStatus(); + return (status == S_CONNECT_FAILED || status == S_CONNECT_LOST || + status == S_DOMAIN_UNRESOLVABLE)? getRetryDelaySeconds() : 0; + // no delay for most + } + + /** + * Update any scheduling structures with the new information in this + * CrawlURI. Chiefly means make necessary arrangements for no other URIs at + * the same host to be visited within the appropriate politeness window. + * + * @param curi + * The CrawlURI + * @return millisecond politeness delay + */ + protected long politenessDelayFor(CrawlURI curi) { + long durationToWait = 0; + Map cdata = curi.getData(); + if (cdata.containsKey(ModuleAttributeConstants.A_FETCH_BEGAN_TIME) + && cdata.containsKey(A_FETCH_COMPLETED_TIME)) { + + long completeTime = curi.getFetchCompletedTime(); + long durationTaken = (completeTime - curi.getFetchBeginTime()); + durationToWait = (long)(getDelayFactor() * durationTaken); + + long minDelay = getMinDelayMs(); + if (minDelay > durationToWait) { + // wait at least the minimum + durationToWait = minDelay; + } + + long maxDelay = getMaxDelayMs(); + if (durationToWait > maxDelay) { + // wait no more than the maximum + durationToWait = maxDelay; + } + + if (getRespectCrawlDelay()) { + CrawlServer s = ServerCacheUtil.getServerFor( + getServerCache(),curi.getUURI()); + UserAgentProvider uap = getUserAgentProvider(); + String ua = curi.getUserAgent(); + if (ua == null) { + ua = uap.getUserAgent(); + } + RobotsExclusionPolicy rep = s.getRobots(); + if (rep != null) { + long crawlDelay = 1000 * s.getRobots().getCrawlDelay(ua); + if (crawlDelay > durationToWait) { + // wait at least the directive crawl-delay + durationToWait = crawlDelay; + } + } + } + + long now = System.currentTimeMillis(); + int maxBandwidthKB = getMaxPerHostBandwidthUsageKbSec(); + if (maxBandwidthKB > 0) { + // Enforce bandwidth limit + ServerCache cache = this.getServerCache(); + CrawlHost host = ServerCacheUtil.getHostFor(cache, curi.getUURI()); + long minDurationToWait = host.getEarliestNextURIEmitTime() + - now; + float maxBandwidth = maxBandwidthKB * 1.024F; // kilo factor + long processedBytes = curi.getContentSize(); + host + .setEarliestNextURIEmitTime((long)(processedBytes / maxBandwidth) + + now); + + if (minDurationToWait > durationToWait) { + durationToWait = minDurationToWait; + } + } + } + return durationToWait; + } + + /** + * Take note of any processor-local errors that have been entered into the + * CrawlURI. + * + * @param curi + * + */ + protected void logNonfatalErrors(CrawlURI curi) { + if (curi.containsDataKey(A_NONFATAL_ERRORS)) { + Collection x = curi.getNonFatalFailures(); + Logger le = loggerModule.getNonfatalErrors(); + for (Throwable e : x) { + le.log(Level.WARNING, curi.toString(), + new Object[] { curi, e }); + } + // once logged, discard + curi.getData().remove(A_NONFATAL_ERRORS); + } + } + + protected boolean overMaxRetries(CrawlURI curi) { + // never retry more than the max number of times + if (curi.getFetchAttempts() >= getMaxRetries()) { + return true; + } + return false; + } + + /* (non-Javadoc) + * @see org.archive.crawler.framework.Frontier#importURIs(java.util.Map) + */ + public void importURIs(String jsonParams) + throws IOException { + JSONObject params; + try { + params = new JSONObject(jsonParams); + } catch (JSONException e) { + IOException ioe = new IOException(e.getMessage()); + ioe.initCause(e); + throw ioe; + } + if("recoveryLog".equals(params.optString("format"))) { + FrontierJournal.importRecoverLog(params, controller); + return; + } + // otherwise, do a 'simple' import + importURIsSimple(params); + } + + /** + * Import URIs from either a simple (one URI per line) or crawl.log + * format. + * + * @param params JSONObject of options to control import + * @see org.archive.crawler.framework.Frontier#importURIs(java.util.Map) + */ + protected void importURIsSimple(JSONObject params) { + // Figure the regex to use parsing each line of input stream. + String extractor; + String output; + String format = params.optString("format"); + if("crawlLog".equals(format)) { + // Skip first 3 fields + extractor = "\\S+\\s+\\S+\\s+\\S+\\s+(\\S+\\s+\\S+\\s+\\S+\\s+).*"; + output = "$1"; + } else { + extractor = + RegexpLineIterator.NONWHITESPACE_ENTRY_TRAILING_COMMENT; + output = RegexpLineIterator.ENTRY; + } + + // Read the input stream. + BufferedReader br = null; + String path = params.optString("path"); + boolean forceRevisit = !params.isNull("forceRevisit"); + boolean asSeeds = !params.isNull("asSeeds"); + boolean scopeScheduleds = !params.isNull("scopeScheduleds"); + DecideRule scope = scopeScheduleds ? getScope() : null; + try { + br = new BufferedReader(new InputStreamReader(new FileInputStream(path))); + Iterator iter = new RegexpLineIterator(new LineReadingIterator(br), + RegexpLineIterator.COMMENT_LINE, extractor, output); + while(iter.hasNext()) { + try { + + CrawlURI curi = CrawlURI.fromHopsViaString(((String)iter.next())); + curi.setForceFetch(forceRevisit); + if (asSeeds) { + curi.setSeed(asSeeds); + if (curi.getVia() == null || curi.getVia().length() <= 0) { + // Danger of double-add of seeds because of this code here. + // Only call addSeed if no via. If a via, the schedule will + // take care of updating scope. + getSeeds().addSeed(curi); + } + } + if(scope!=null) { + //TODO:SPRINGY +// curi.setStateProvider(controller.getSheetManager()); + if(!scope.accepts(curi)) { + continue; + } + } + + this.controller.getFrontier().schedule(curi); + + } catch (URIException e) { + e.printStackTrace(); + } + } + br.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + /** + * Log to the main crawl.log + * + * @param curi + */ + protected void log(CrawlURI curi) { + curi.aboutToLog(); + Object array[] = {curi}; + this.loggerModule.getUriProcessing().log(Level.INFO, + curi.getUURI().toString(), array); + } + + protected boolean isDisregarded(CrawlURI curi) { + switch (curi.getFetchStatus()) { + case S_ROBOTS_PRECLUDED: // they don't want us to have it + case S_BLOCKED_BY_CUSTOM_PROCESSOR: + case S_OUT_OF_SCOPE: // filtered out by scope + case S_BLOCKED_BY_USER: // filtered out by user + case S_TOO_MANY_EMBED_HOPS: // too far from last true link + case S_TOO_MANY_LINK_HOPS: // too far from seeds + case S_DELETED_BY_USER: // user deleted + return true; + default: + return false; + } + } + + /** + * Checks if a recently completed CrawlURI that did not finish successfully + * needs to be retried (processed again after some time elapses) + * + * @param curi + * The CrawlURI to check + * @return True if we need to retry. + */ + protected boolean needsRetrying(CrawlURI curi) { + if (overMaxRetries(curi)) { + return false; + } + + switch (curi.getFetchStatus()) { + case HttpStatus.SC_UNAUTHORIZED: + // We can get here though usually a positive status code is + // a success. We get here if there is rfc2617 credential data + // loaded and we're supposed to go around again. See if any + // rfc2617 credential present and if there, assume it got + // loaded in FetchHTTP on expectation that we're to go around + // again. If no rfc2617 loaded, we should not be here. + boolean loaded = curi.hasRfc2617CredentialAvatar(); + if (!loaded && logger.isLoggable(Level.INFO)) { + logger.info("Have 401 but no creds loaded " + curi); + } + return loaded; + case S_DEFERRED: + case S_CONNECT_FAILED: + case S_CONNECT_LOST: + case S_DOMAIN_UNRESOLVABLE: + // these are all worth a retry + // TODO: consider if any others (S_TIMEOUT in some cases?) deserve + // retry + return true; + default: + return false; + } + } + + /** + * Canonicalize passed uuri. Its would be sweeter if this canonicalize + * function was encapsulated by that which it canonicalizes but because + * settings change with context -- i.e. there may be overrides in operation + * for a particular URI -- its not so easy; Each CrawlURI would need a + * reference to the settings system. That's awkward to pass in. + * + * @param uuri Candidate URI to canonicalize. + * @return Canonicalized version of passed uuri. + */ + protected String canonicalize(UURI uuri) { + return getUriCanonicalizationPolicy().canonicalize(uuri.toString()); + } + + /** + * Canonicalize passed CrawlURI. This method differs from + * {@link #canonicalize(UURI)} in that it takes a look at + * the CrawlURI context possibly overriding any canonicalization effect if + * it could make us miss content. If canonicalization produces an URL that + * was 'alreadyseen', but the entry in the 'alreadyseen' database did + * nothing but redirect to the current URL, we won't get the current URL; + * we'll think we've already see it. Examples would be archive.org + * redirecting to www.archive.org or the inverse, www.netarkivet.net + * redirecting to netarkivet.net (assuming stripWWW rule enabled). + *

Note, this method under circumstance sets the forceFetch flag. + * + * @param cauri CrawlURI to examine. + * @return Canonicalized cacuri. + */ + protected String canonicalize(CrawlURI cauri) { + String canon = canonicalize(cauri.getUURI()); + if (cauri.isLocation()) { + // If the via is not the same as where we're being redirected (i.e. + // we're not being redirected back to the same page, AND the + // canonicalization of the via is equal to the the current cauri, + // THEN forcefetch (Forcefetch so no chance of our not crawling + // content because alreadyseen check things its seen the url before. + // An example of an URL that redirects to itself is: + // http://bridalelegance.com/images/buttons3/tuxedos-off.gif. + // An example of an URL whose canonicalization equals its via's + // canonicalization, and we want to fetch content at the + // redirection (i.e. need to set forcefetch), is netarkivet.dk. + if (!cauri.toString().equals(cauri.getVia().toString()) && + canonicalize(cauri.getVia()).equals(canon)) { + cauri.setForceFetch(true); + } + } + return canon; + } + + /** + * @param cauri CrawlURI we're to get a key for. + * @return a String token representing a queue + */ + public String getClassKey(CrawlURI curi) { + assert KeyedProperties.overridesActiveFrom(curi); + + String queueKey = getForceQueueAssignment(); + if ("".equals(queueKey)) { + // Typical case, barring overrides + //TODO:SPRINGY set overrides based on cauri? + queueKey = getQueueAssignmentPolicy().getClassKey(curi); + } + return queueKey; + } + + /** + * @return RecoveryJournal instance. May be null. + */ + public FrontierJournal getFrontierJournal() { + return this.recover; + } + + public void crawlEnded(String sExitMessage) { + if (logger.isLoggable(Level.INFO)) { + logger.info("Closing with " + Long.toString(queuedUriCount()) + + " urls still in queue."); + } + } + + // + // Reporter implementation + // + public String singleLineReport() { + return ArchiveUtils.singleLineReport(this); + } + + public void reportTo(PrintWriter writer) { + reportTo(null, writer); + } + + + private void writeObject(ObjectOutputStream out) + throws IOException { + out.defaultWriteObject(); + boolean recoveryLogEnabled = getRecoveryLogEnabled(); + out.writeBoolean(recoveryLogEnabled); + if (recoveryLogEnabled) { + out.writeUTF(loggerModule.getPath().getFile().getAbsolutePath()); + } + } + + + private void readObject(ObjectInputStream inp) + throws IOException, ClassNotFoundException { + inp.defaultReadObject(); + boolean recoveryLogEnabled = inp.readBoolean(); + if (recoveryLogEnabled) { + String path = inp.readUTF(); + if (inp instanceof CheckpointRecovery) { + CheckpointRecovery cr = (CheckpointRecovery)inp; + path = cr.translatePath(path); + new File(path).mkdirs(); + } + initJournal(path); + } + targetState = State.PAUSE; + outbound = new ArrayBlockingQueue(outboundCapacity, true); + inbound = new ArrayBlockingQueue(inboundCapacity, true); + } + + /** + * Arrange for the given InEvent to be done by the managerThread, via + * enqueueing with other events if possible, but directly if not possible + * and this is the managerThread. + * @param ev InEvent to be done + */ + protected void enqueueOrDo(InEvent ev) { + if(!inbound.offer(ev)) { + // if can't defer, + if(Thread.currentThread()==managerThread) { + // if can't enqueue, ok to just do + ev.process(); + return; + } else { + try { + inbound.put(ev); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + } + + /** + * Arrange for the given InEvent to be done by the managerThread, + * immediately if this is the managerThread, of via enqueueing with + * other inbound events otherwise. + * @param ev InEvent to be done + */ + protected void doOrEnqueue(InEvent ev) { + if (Thread.currentThread() == managerThread) { + // if can't enqueue, ok to just do + ev.process(); + return; + } else { + try { + inbound.put(ev); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + + /** + * An event/update for the managerThread to process from the inbound queue. + */ + public abstract class InEvent { + abstract public void process(); + } + + /** + * A CrawlURI to be scheduled by the managerThread without regard to + * whether the CrawlURI was already-seen. + */ + public class ScheduleAlways extends InEvent { + CrawlURI curi; + public ScheduleAlways(CrawlURI c) { + this.curi = c; + } + public void process() { + try { + KeyedProperties.loadOverridesFrom(curi); + processScheduleAlways(curi); + } finally { + KeyedProperties.clearOverridesFrom(curi); + } + } + } + + /** + * A CrawlURI to be scheduled by the managerThread if it has not been + * already-seen. (That is, if it passes the UriUniqFilter.) + */ + public class ScheduleIfUnique extends InEvent { + CrawlURI curi; + public ScheduleIfUnique(CrawlURI c) { + this.curi = c; + } + public void process() { + try { + KeyedProperties.loadOverridesFrom(curi); + processScheduleIfUnique(curi); + } finally { + KeyedProperties.clearOverridesFrom(curi); + } + } + } + + /** + * A CrawlURI, previously issued via the outbound queue, that has finished + * its processing chain with update implications for the frontier state. + */ + public class Finish extends InEvent { + CrawlURI caUri; + public Finish(CrawlURI c) { + this.caUri = c; + } + public void process() { + try { + KeyedProperties.loadOverridesFrom(caUri); + processFinish(caUri); + } finally { + KeyedProperties.clearOverridesFrom(caUri); + } + } + } + + /** + * An request that the frontier enter a new Frontier.State. + */ + public class SetTargetState extends InEvent { + State target; + public SetTargetState(State target) { + this.target = target; + } + @Override + public void process() { + processSetTargetState(target); + // TODO: perhaps null reachedState, because until new state is + // reached it's misleading? + } + } + + public void onApplicationEvent(ApplicationEvent event) { + if(event instanceof CrawlStateEvent) { + CrawlStateEvent event1 = (CrawlStateEvent)event; + switch(event1.getState()) { + case FINISHED: + this.crawlEnded(event1.getMessage()); + break; + default: + // ignore; + } + } + } + + @SuppressWarnings("unchecked") + protected void applyOverridesTo(CrawlURI curi) { + curi.setOverlayMapsSource(sheetOverlaysManager); + if(!curi.haveOverlayNamesBeenSet()) { + sheetOverlaysManager.applyOverlays(curi); + } + } +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/AntiCalendarCostAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/AntiCalendarCostAssignmentPolicy.java new file mode 100644 index 00000000..6a622f65 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/AntiCalendarCostAssignmentPolicy.java @@ -0,0 +1,65 @@ +/* AntiCalendarCostAssignmentPolicy +* +* $Id$ +* +* Created on Dec 15, 2004 +* +* Copyright (C) 2004 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.crawler.frontier; + +import java.util.regex.Matcher; + +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.util.TextUtils; + +/** + * CostAssignmentPolicy that further penalizes URIs with + * calendar-suggestive strings in them, with an extra unit + * of cost. + * + * Will catch some 'innocent' URIs, but only when uncaught + * large-volume chaff is ranked higher than caught 'wheat' + * will this cause notable problems. + * + * @author gojomo + */ +public class AntiCalendarCostAssignmentPolicy extends UnitCostAssignmentPolicy { + + private static final long serialVersionUID = 3L; + + public static String CALENDARISH = + "(?i)(calendar)|(year)|(month)|(day)|(date)|(viewcal)" + + "|(\\D19\\d\\d\\D)|(\\D20\\d\\d\\D)|(event)|(yr=)" + + "|(calendrier)|(jour)"; + + /* (non-Javadoc) + * @see org.archive.crawler.frontier.CostAssignmentPolicy#costOf(org.archive.crawler.datamodel.CrawlURI) + */ + public int costOf(CrawlURI curi) { + int cost = super.costOf(curi); + Matcher m = TextUtils.getMatcher(CALENDARISH, curi.toString()); + if (m.find()) { + cost++; + // TODO: consider if multiple occurences should cost more + } + TextUtils.recycleMatcher(m); + return cost; + } +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java new file mode 100644 index 00000000..03246ce1 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java @@ -0,0 +1,336 @@ +/* BdbFrontier + * + * $Id$ +* + * Created on Sep 24, 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ +package org.archive.crawler.frontier; + +import java.io.File; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.Serializable; +import java.util.Iterator; +import java.util.List; +import java.util.Queue; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.concurrent.DelayQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Pattern; + +import javax.management.openmbean.CompositeData; + +import org.apache.commons.collections.Closure; +import org.archive.bdb.BdbModule; +import org.archive.checkpointing.Checkpointable; +import org.archive.checkpointing.RecoverAction; +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.queue.StoredQueue; +import org.springframework.beans.factory.annotation.Autowired; + +import com.sleepycat.collections.StoredIterator; +import com.sleepycat.je.Database; +import com.sleepycat.je.DatabaseException; + +/** + * A Frontier using several BerkeleyDB JE Databases to hold its record of + * known hosts (queues), and pending URIs. + * + * @author Gordon Mohr + */ +public class BdbFrontier extends WorkQueueFrontier +implements Serializable, Checkpointable { + private static final long serialVersionUID = 1L; + + private static final Logger logger = + Logger.getLogger(BdbFrontier.class.getName()); + + /** + * All 'inactive' queues, not yet in active rotation. + * Linked-list of keys for the queues. + */ + protected SortedMap> inactiveQueuesByPrecedence; + + /** + * 'retired' queues, no longer considered for activation. + * Linked-list of keys for queues. + */ + protected StoredQueue retiredQueues; + + /** all URIs scheduled to be crawled */ + protected transient BdbMultipleWorkQueues pendingUris; + + protected BdbModule bdb; + @Autowired + public void setBdbModule(BdbModule bdb) { + this.bdb = bdb; + } + + boolean dumpPendingAtClose = false; + public boolean getDumpPendingAtClose() { + return dumpPendingAtClose; + } + public void setDumpPendingAtClose(boolean dumpPendingAtClose) { + this.dumpPendingAtClose = dumpPendingAtClose; + } + + /* (non-Javadoc) + * @see org.archive.crawler.frontier.WorkQueueFrontier#getInactiveQueuesByPrecedence() + */ + @Override + SortedMap> getInactiveQueuesByPrecedence() { + return inactiveQueuesByPrecedence; + } + + /* (non-Javadoc) + * @see org.archive.crawler.frontier.WorkQueueFrontier#getRetiredQueues() + */ + @Override + Queue getRetiredQueues() { + return retiredQueues; + } + + /** + * Create the single object (within which is one BDB database) + * inside which all the other queues live. + * + * @return the created BdbMultipleWorkQueues + * @throws DatabaseException + */ + private BdbMultipleWorkQueues createMultipleWorkQueues(boolean recycle) + throws DatabaseException { + Database db; + if (recycle) { + db = bdb.getDatabase("pending"); + } else { + BdbModule.BdbConfig dbConfig = new BdbModule.BdbConfig(); + dbConfig.setAllowCreate(!recycle); + // Make database deferred write: URLs that are added then removed + // before a page-out is required need never cause disk IO. + db = bdb.openDatabase("pending", dbConfig, recycle); + } + + return new BdbMultipleWorkQueues(db, bdb.getClassCatalog()); + } + + + /** + * Return the work queue for the given CrawlURI's classKey. URIs + * are ordered and politeness-delayed within their 'class'. + * + * @param curi CrawlURI to base queue on + * @return the found or created BdbWorkQueue + */ + protected WorkQueue getQueueFor(CrawlURI curi) { + WorkQueue wq; + String classKey = curi.getClassKey(); + synchronized (allQueues) { + wq = (WorkQueue)allQueues.get(classKey); + if (wq == null) { + wq = new BdbWorkQueue(classKey, this); + //TODO:SPRINGY set overrides + wq.setTotalBudget(getQueueTotalBudget()); + //TODO:SPRINGY set overrides + getQueuePrecedencePolicy().queueCreated(wq); + allQueues.put(classKey, wq); + } + } + return wq; + } + + /** + * Return the work queue for the given classKey, or null + * if no such queue exists. + * + * @param classKey key to look for + * @return the found WorkQueue + */ + protected WorkQueue getQueueFor(String classKey) { + assert Thread.currentThread() == managerThread; + + WorkQueue wq = (WorkQueue)allQueues.get(classKey); + return wq; + } + + + /** + * Return list of urls. + * @param marker + * @param numberOfMatches + * @param verbose + * @return List of URIs (strings). + */ + public CompositeData getURIsList(String marker, + int numberOfMatches, String pattern, final boolean verbose) { + try { + Pattern p = Pattern.compile(pattern); + return pendingUris.getFrom(marker, numberOfMatches, p, verbose); + } catch (DatabaseException e) { + throw new IllegalStateException(e); + } + } + + protected void closeQueue() { + if (getDumpPendingAtClose()) { + try { + dumpAllPendingToLog(); + } catch (DatabaseException e) { + logger.log(Level.WARNING, "dump pending problem", e); + } + } + if (this.pendingUris != null) { + this.pendingUris.close(); + this.pendingUris = null; + } + } + + protected BdbMultipleWorkQueues getWorkQueues() { + return pendingUris; + } + + protected boolean workQueueDataOnDisk() { + return true; + } + + + /** + * Constructor. + */ + public BdbFrontier() { + super(); + } + + public void checkpoint(File checkpointDir, List actions) + throws IOException { + logger.fine("Started syncing already seen as part " + + "of checkpoint. Can take some time."); + // An explicit sync on the any deferred write dbs is needed to make the + // db recoverable. Sync'ing the environment doesn't work. + if (this.pendingUris != null) { + this.pendingUris.sync(); + } + logger.fine("Finished syncing already seen as part of checkpoint."); + } + + + @Override + protected void initAllQueues() throws DatabaseException { + this.allQueues = bdb.getBigMap("allqueues", false, + String.class, WorkQueue.class); + if (logger.isLoggable(Level.FINE)) { + Iterator i = this.allQueues.keySet().iterator(); + try { + for (; i.hasNext();) { + logger.fine((String) i.next()); + } + } finally { + StoredIterator.close(i); + } + } + } + + @Override + protected void initOtherQueues(boolean recycle) throws DatabaseException { + // small risk of OutOfMemoryError: if 'hold-queues' is false, + // readyClassQueues may grow in size without bound + readyClassQueues = new LinkedBlockingQueue(); + + inactiveQueuesByPrecedence = new TreeMap>(); + + Database retiredQueuesDb; + retiredQueuesDb = bdb.openDatabase("retiredQueues", + StoredQueue.databaseConfig(), false); + retiredQueues = new StoredQueue(retiredQueuesDb, + String.class, null); + + // small risk of OutOfMemoryError: in large crawls with many + // unresponsive queues, an unbounded number of snoozed queues + // may exist + snoozedClassQueues = new DelayQueue(); + + // initialize master map in which other queues live + this.pendingUris = createMultipleWorkQueues(recycle); + } + + + /* (non-Javadoc) + * @see org.archive.crawler.frontier.WorkQueueFrontier#createInactiveQueueForPrecedence(int) + */ + @Override + Queue createInactiveQueueForPrecedence(int precedence) { + Database inactiveQueuesDb; + try { + inactiveQueuesDb = bdb.openDatabase("inactiveQueues-"+precedence, + StoredQueue.databaseConfig(), false); + } catch (DatabaseException e) { + throw new RuntimeException(e); + } + return new StoredQueue(inactiveQueuesDb, + String.class, null); + } + + private void readObject(ObjectInputStream in) + throws IOException, ClassNotFoundException { + in.defaultReadObject(); + + // rehook StoredQueues to their databases + for(int precedenceKey : inactiveQueuesByPrecedence.keySet()) { + Database inactiveQueuesDb = + bdb.getDatabase("inactiveQueues-"+precedenceKey); + ((StoredQueue)inactiveQueuesByPrecedence.get(precedenceKey)) + .hookupDatabase(inactiveQueuesDb, String.class, null); + } + + // rehook retiredQueues to its database + Database retiredQueuesDb = bdb.getDatabase("retiredQueues"); + retiredQueues.hookupDatabase(retiredQueuesDb, String.class, null); + + try { + this.pendingUris = new BdbMultipleWorkQueues(bdb.getDatabase("pending"), + bdb.getClassCatalog()); + } catch (DatabaseException e) { + IOException io = new IOException(); + io.initCause(e); + throw io; + } + startManagerThread(); + } + + /** + * Dump all still-enqueued URIs to the crawl.log -- without actually + * dequeuing. Useful for understanding what was remaining in a crawl that + * was ended early, for example at a time limit. + * + * @throws DatabaseException + */ + public void dumpAllPendingToLog() throws DatabaseException { + Closure tolog = new Closure() { + public void execute(Object curi) { + log((CrawlURI) curi); + } + }; + pendingUris.forAllPendingDo(tolog); + } +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java b/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java new file mode 100644 index 00000000..cebfe549 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java @@ -0,0 +1,561 @@ +/* BdbMultipleWorkQueues + * + * Created on Dec 24, 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.crawler.frontier; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Pattern; + +import javax.management.openmbean.CompositeData; +import javax.management.openmbean.CompositeDataSupport; +import javax.management.openmbean.OpenDataException; + +import org.apache.commons.collections.Closure; +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.util.ArchiveUtils; + +import com.sleepycat.bind.serial.StoredClassCatalog; +import com.sleepycat.je.Cursor; +import com.sleepycat.je.Database; +import com.sleepycat.je.DatabaseEntry; +import com.sleepycat.je.DatabaseException; +import com.sleepycat.je.OperationStatus; +import com.sleepycat.util.RuntimeExceptionWrapper; + + +/** + * A BerkeleyDB-database-backed structure for holding ordered + * groupings of CrawlURIs. Reading the groupings from specific + * per-grouping (per-classKey/per-Host) starting points allows + * this to act as a collection of independent queues. + * + *

For how the bdb keys are made, see {@link #calculateInsertKey(CrawlURI)}. + * + *

TODO: refactor, improve naming. + * + * @author gojomo + */ +public class BdbMultipleWorkQueues { + private static final long serialVersionUID = 1L; + + private static final Logger LOGGER = + Logger.getLogger(BdbMultipleWorkQueues.class.getName()); + + /** Database holding all pending URIs, grouped in virtual queues */ + private Database pendingUrisDB = null; + + /** Supporting bdb serialization of CrawlURIs */ + private RecyclingSerialBinding crawlUriBinding; + + /** + * Create the multi queue in the given environment. + * + * @param env bdb environment to use + * @param classCatalog Class catalog to use. + * @param recycle True if we are to reuse db content if any. + * @throws DatabaseException + */ + public BdbMultipleWorkQueues(Database db, + StoredClassCatalog classCatalog) + throws DatabaseException { + this.pendingUrisDB = db; + crawlUriBinding = + new RecyclingSerialBinding(classCatalog, CrawlURI.class); + } + + /** + * Delete all CrawlURIs matching the given expression. + * + * @param match + * @param queue + * @param headKey + * @return count of deleted items + * @throws DatabaseException + * @throws DatabaseException + */ + public long deleteMatchingFromQueue(String match, String queue, + DatabaseEntry headKey) throws DatabaseException { + long deletedCount = 0; + Pattern pattern = Pattern.compile(match); + DatabaseEntry key = headKey; + DatabaseEntry value = new DatabaseEntry(); + Cursor cursor = null; + try { + cursor = pendingUrisDB.openCursor(null, null); + OperationStatus result = cursor.getSearchKeyRange(headKey, + value, null); + + while (result == OperationStatus.SUCCESS) { + if(value.getData().length>0) { + CrawlURI curi = (CrawlURI) crawlUriBinding + .entryToObject(value); + if (!curi.getClassKey().equals(queue)) { + // rolled into next queue; finished with this queue + break; + } + if (pattern.matcher(curi.toString()).matches()) { + cursor.delete(); + deletedCount++; + } + } + result = cursor.getNext(key, value, null); + } + } finally { + if (cursor != null) { + cursor.close(); + } + } + + return deletedCount; + } + + /** + * @param m marker + * @param maxMatches + * @return list of matches starting from marker position + * @throws DatabaseException + */ + public CompositeData getFrom( + String m, + int maxMatches, + Pattern pattern, + boolean verbose) + throws DatabaseException { + int matches = 0; + int tries = 0; + ArrayList results = new ArrayList(maxMatches); + byte[] marker = m.getBytes(); // = FrontierJMXTypes.fromString(m); + + DatabaseEntry key; + if (marker == null) { + key = getFirstKey(); + } else { + key = new DatabaseEntry(marker); + } + + DatabaseEntry value = new DatabaseEntry(); + + Cursor cursor = null; + OperationStatus result = null; + try { + cursor = pendingUrisDB.openCursor(null,null); + result = cursor.getSearchKey(key, value, null); + + while(matches < maxMatches && result == OperationStatus.SUCCESS) { + if(value.getData().length>0) { + CrawlURI curi = (CrawlURI) crawlUriBinding.entryToObject(value); + if(pattern.matcher(curi.toString()).matches()) { + if (verbose) { + results.add("[" + curi.getClassKey() + "] " + + curi.singleLineReport()); + } else { + results.add(curi.toString()); + } + matches++; + } + tries++; + } + result = cursor.getNext(key,value,null); + } + } finally { + if (cursor !=null) { + cursor.close(); + } + } + + if(result != OperationStatus.SUCCESS) { + // end of scan + m = null; + } else { + m = new String(key.getData()); // = FrontierJMXTypes.toString(key.getData()); + } + + String[] arr = results.toArray(new String[results.size()]); + CompositeData cd; + try { + cd = new CompositeDataSupport( + /*FrontierJMXTypes.URI_LIST_DATA*/ null, + new String[] { "list", "marker" }, + new Object[] { arr, m }); + } catch (OpenDataException e) { + throw new IllegalStateException(e); + } + return cd; + } + + /** + * @return the key to the first item in the database + * @throws DatabaseException + */ + protected DatabaseEntry getFirstKey() throws DatabaseException { + DatabaseEntry key = new DatabaseEntry(); + DatabaseEntry value = new DatabaseEntry(); + Cursor cursor = pendingUrisDB.openCursor(null,null); + OperationStatus status = cursor.getNext(key,value,null); + cursor.close(); + if(status == OperationStatus.SUCCESS) { + return key; + } + return null; + } + + /** + * Get the next nearest item after the given key. Relies on + * external discipline -- we'll look at the queues count of how many + * items it has -- to avoid asking for something from a + * range where there are no associated items -- + * otherwise could get first item of next 'queue' by mistake. + * + *

TODO: hold within a queue's range + * + * @param headKey Key prefix that demarks the beginning of the range + * in pendingUrisDB we're interested in. + * @return CrawlURI. + * @throws DatabaseException + */ + public CrawlURI get(DatabaseEntry headKey) + throws DatabaseException { + DatabaseEntry result = new DatabaseEntry(); + + // From Linda Lee of sleepycat: + // "You want to check the status returned from Cursor.getSearchKeyRange + // to make sure that you have OperationStatus.SUCCESS. In that case, + // you have found a valid data record, and result.getData() + // (called by internally by the binding code, in this case) will be + // non-null. The other possible status return is + // OperationStatus.NOTFOUND, in which case no data record matched + // the criteria. " + OperationStatus status = getNextNearestItem(headKey, result); + CrawlURI retVal = null; + if (status != OperationStatus.SUCCESS) { + LOGGER.severe("See '1219854 NPE je-2.0 " + + "entryToObject...'. OperationStatus " + + " was not SUCCESS: " + + status + + ", headKey " + + BdbWorkQueue.getPrefixClassKey(headKey.getData())); + return null; + } + + try { + retVal = (CrawlURI)crawlUriBinding.entryToObject(result); + } catch (ClassCastException cce) { + Object obj = crawlUriBinding.entryToObject(result); + LOGGER.log(Level.SEVERE, + "see [#HER-1283]: deserialized " + obj.getClass() + + " has ClassLoader " + + obj.getClass().getClassLoader().getClass(), + cce); + return null; + } catch (RuntimeExceptionWrapper rw) { + LOGGER.log( + Level.SEVERE, + "expected object missing in queue " + + BdbWorkQueue.getPrefixClassKey(headKey.getData()), + rw); + return null; + } + retVal.setHolderKey(headKey); + return retVal; + } + + protected OperationStatus getNextNearestItem(DatabaseEntry headKey, + DatabaseEntry result) throws DatabaseException { + Cursor cursor = null; + OperationStatus status; + try { + cursor = this.pendingUrisDB.openCursor(null, null); + + // get cap; headKey at this point should always point to + // a queue-beginning cap entry (zero-length value) + status = cursor.getSearchKey(headKey, result, null); + if (status != OperationStatus.SUCCESS) { + throw new DatabaseException("bdb queue cap missing: " + + status.toString() + " " + new String(headKey.getData())); + } + if (result.getData().length > 0) { + throw new DatabaseException("bdb queue has nonzero size: " + + result.getData().length); + } + // get next item (real first item of queue) + status = cursor.getNext(headKey,result,null); + } finally { + if(cursor!=null) { + cursor.close(); + } + } + return status; + } + + + /** + * Put the given CrawlURI in at the appropriate place. + * + * @param curi + * @throws DatabaseException + */ + public void put(CrawlURI curi, boolean overwriteIfPresent) + throws DatabaseException { + DatabaseEntry insertKey = (DatabaseEntry)curi.getHolderKey(); + if (insertKey == null) { + insertKey = calculateInsertKey(curi); + curi.setHolderKey(insertKey); + } + DatabaseEntry value = new DatabaseEntry(); + crawlUriBinding.objectToEntry(curi, value); + // Output tally on avg. size if level is FINE or greater. + if (LOGGER.isLoggable(Level.FINE)) { + tallyAverageEntrySize(curi, value); + } + OperationStatus status; + if(overwriteIfPresent) { + status = pendingUrisDB.put(null, insertKey, value); + } else { + status = pendingUrisDB.putNoOverwrite(null, insertKey, value); + } + + if (status!=OperationStatus.SUCCESS) { + LOGGER.severe("failed; "+status+ " "+curi); + } + } + + private long entryCount = 0; + private long entrySizeSum = 0; + private int largestEntry = 0; + + /** + * Log average size of database entry. + * @param curi CrawlURI this entry is for. + * @param value Database entry value. + */ + private synchronized void tallyAverageEntrySize(CrawlURI curi, + DatabaseEntry value) { + entryCount++; + int length = value.getData().length; + entrySizeSum += length; + int avg = (int) (entrySizeSum/entryCount); + if(entryCount % 1000 == 0) { + LOGGER.fine("Average entry size at "+entryCount+": "+avg); + } + if (length>largestEntry) { + largestEntry = length; + LOGGER.fine("Largest entry: "+length+" "+curi); + if(length>(2*avg)) { + LOGGER.fine("excessive?"); + } + } + } + + /** + * Calculate the 'origin' key for a virtual queue of items + * with the given classKey. This origin key will be a + * prefix of the keys for all items in the queue. + * + * @param classKey String key to derive origin byte key from + * @return a byte array key + */ + static byte[] calculateOriginKey(String classKey) { + byte[] classKeyBytes = null; + int len = 0; + try { + classKeyBytes = classKey.getBytes("UTF-8"); + len = classKeyBytes.length; + } catch (UnsupportedEncodingException e) { + // should be impossible; all JVMs must support UTF-8 + e.printStackTrace(); + } + byte[] keyData = new byte[len+1]; + System.arraycopy(classKeyBytes,0,keyData,0,len); + keyData[len]=0; + return keyData; + } + + /** + * Calculate the insertKey that places a CrawlURI in the + * desired spot. First bytes are always classKey (usu. host) + * based -- ensuring grouping by host -- terminated by a zero + * byte. Then 8 bytes of data ensuring desired ordering + * within that 'queue' are used. The first byte of these 8 is + * priority -- allowing 'immediate' and 'soon' items to + * sort above regular. Next 1 byte is 'precedence'. Last 6 bytes + * are ordinal serial number, ensuring earlier-discovered + * URIs sort before later. + * + * NOTE: Dangers here are: + * (1) priorities or precedences over 2^7 (signed byte comparison) + * (2) ordinals over 2^48 + * + * Package access & static for testing purposes. + * + * @param curi + * @return a DatabaseEntry key for the CrawlURI + */ + static DatabaseEntry calculateInsertKey(CrawlURI curi) { + byte[] classKeyBytes = null; + int len = 0; + try { + classKeyBytes = curi.getClassKey().getBytes("UTF-8"); + len = classKeyBytes.length; + } catch (UnsupportedEncodingException e) { + // should be impossible; all JVMs must support UTF-8 + e.printStackTrace(); + } + byte[] keyData = new byte[len+9]; + System.arraycopy(classKeyBytes,0,keyData,0,len); + keyData[len]=0; + long ordinalPlus = curi.getOrdinal() & 0x0000FFFFFFFFFFFFL; + ordinalPlus = + ((long)curi.getSchedulingDirective() << 56) | ordinalPlus; + long precedence = Math.min(curi.getPrecedence(), 127); + ordinalPlus = + (((precedence) & 0xFFL) << 48) | ordinalPlus; + ArchiveUtils.longIntoByteArray(ordinalPlus, keyData, len+1); + return new DatabaseEntry(keyData); + } + + + static String insertKeyToString(DatabaseEntry holderKey) { + StringBuilder result = new StringBuilder(); + byte[] data = holderKey.getData(); + int p = findFirstZero(data); + result.append(new String(data, 0, p)); + + java.io.ByteArrayInputStream binp = + new java.io.ByteArrayInputStream(data, p + 1, data.length); + java.io.DataInputStream dinp = new java.io.DataInputStream(binp); + long l = 0; + try { + l = dinp.readLong(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + result.append(" blah=").append(l); + + return result.toString(); + } + + + private static int findFirstZero(byte[] b) { + for (int i = 0; i < b.length; i++) { + if (b[i] == 0) { + return i; + } + } + return -1; + } + + /** + * Delete the given CrawlURI from persistent store. Requires + * the key under which it was stored be available. + * + * @param item + * @throws DatabaseException + */ + public void delete(CrawlURI item) throws DatabaseException { + OperationStatus status; + DatabaseEntry de = (DatabaseEntry)item.getHolderKey(); + status = pendingUrisDB.delete(null, de); + if (status != OperationStatus.SUCCESS) { + LOGGER.severe("expected item not present: " + + item + + "(" + + (new BigInteger(((DatabaseEntry) item.getHolderKey()) + .getData())).toString(16) + ")"); + } + } + + /** + * Method used by BdbFrontier during checkpointing. + *

The backing bdbje database has been marked deferred write so we save + * on writes to disk. Means no guarantees disk will have whats in memory + * unless a sync is called (Calling sync on the bdbje Environment is not + * sufficent). + *

Package access only because only Frontiers of this package would ever + * need access. + * @see Deferred Write Databases + */ + void sync() { + if (this.pendingUrisDB == null) { + return; + } + try { + this.pendingUrisDB.sync(); + } catch (DatabaseException e) { + e.printStackTrace(); + } + } + + /** + * clean up + * + */ + public void close() { +/* try { + this.pendingUrisDB.close(); + } catch (DatabaseException e) { + e.printStackTrace(); + } */ + } + + + /** + * Add a dummy 'cap' entry at the given insertion key. Prevents + * 'seeks' to queue heads from holding lock on last item of + * 'preceding' queue. See: + * http://sourceforge.net/tracker/index.php?func=detail&aid=1262665&group_id=73833&atid=539102 + * + * @param origin key at which to insert the cap + */ + public void addCap(byte[] origin) { + try { + pendingUrisDB.put(null, new DatabaseEntry(origin), + new DatabaseEntry(new byte[0])); + } catch (DatabaseException e) { + throw new RuntimeException(e); + } + } + + /** + * Utility method to perform action for all pending CrawlURI instances. + * @param c Closure action to perform + * @throws DatabaseException + */ + protected void forAllPendingDo(Closure c) throws DatabaseException { + DatabaseEntry key = new DatabaseEntry(); + DatabaseEntry value = new DatabaseEntry(); + Cursor cursor = pendingUrisDB.openCursor(null, null); + while (cursor.getNext(key, value, null) == OperationStatus.SUCCESS) { + if (value.getData().length == 0) { + continue; + } + CrawlURI item = (CrawlURI) crawlUriBinding.entryToObject(value); + c.execute(item); + } + } +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/BdbWorkQueue.java b/engine/src/main/java/org/archive/crawler/frontier/BdbWorkQueue.java new file mode 100644 index 00000000..bc2a28fa --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/BdbWorkQueue.java @@ -0,0 +1,175 @@ +/* BdbWorkQueue + * + * Created on Dec 24, 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.crawler.frontier; + +import java.io.IOException; +import java.io.Serializable; +import java.io.UnsupportedEncodingException; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.util.ArchiveUtils; +import org.archive.util.IoUtils; + +import com.sleepycat.je.DatabaseEntry; +import com.sleepycat.je.DatabaseException; + + +/** + * One independent queue of items with the same 'classKey' (eg host). + * @author gojomo + */ +public class BdbWorkQueue extends WorkQueue +implements Serializable { + private static final long serialVersionUID = 1L; + private static Logger LOGGER = + Logger.getLogger(BdbWorkQueue.class.getName()); + + + /** + * All items in this queue have this same 'origin' + * prefix to their keys. + */ + private byte[] origin; + + /** + * Create a virtual queue inside the given BdbMultipleWorkQueues + * + * @param classKey + */ + public BdbWorkQueue(String classKey, BdbFrontier frontier) { + super(classKey); + this.origin = BdbMultipleWorkQueues.calculateOriginKey(classKey); + if (LOGGER.isLoggable(Level.FINE)) { + LOGGER.fine(getPrefixClassKey(this.origin) + " " + classKey); + } + // add the queue-front 'cap' entry; see... + // http://sourceforge.net/tracker/index.php?func=detail&aid=1262665&group_id=73833&atid=539102 + frontier.getWorkQueues().addCap(origin); + } + + protected long deleteMatchingFromQueue(final WorkQueueFrontier frontier, + final String match) throws IOException { + try { + final BdbMultipleWorkQueues queues = ((BdbFrontier) frontier) + .getWorkQueues(); + return queues.deleteMatchingFromQueue(match, classKey, + new DatabaseEntry(origin)); + } catch (DatabaseException e) { + throw IoUtils.wrapAsIOException(e); + } + } + + protected void deleteItem(final WorkQueueFrontier frontier, + final CrawlURI peekItem) throws IOException { + try { + final BdbMultipleWorkQueues queues = ((BdbFrontier) frontier) + .getWorkQueues(); + queues.delete(peekItem); + } catch (DatabaseException e) { + e.printStackTrace(); + throw IoUtils.wrapAsIOException(e); + } + } + + protected CrawlURI peekItem(final WorkQueueFrontier frontier) + throws IOException { + final BdbMultipleWorkQueues queues = ((BdbFrontier) frontier) + .getWorkQueues(); + DatabaseEntry key = new DatabaseEntry(origin); + CrawlURI curi = null; + int tries = 1; + while(true) { + try { + curi = queues.get(key); + } catch (DatabaseException e) { + LOGGER.log(Level.SEVERE,"peekItem failure; retrying",e); + } + + // ensure CrawlURI, if any, came from acceptable range: + if(!ArchiveUtils.startsWith(key.getData(),origin)) { + LOGGER.severe( + "inconsistency: "+classKey+"("+ + getPrefixClassKey(origin)+") with " + getCount() + " items gave " + + curi +"("+getPrefixClassKey(key.getData())); + // clear curi to allow retry + curi = null; + // reset key to original origin for retry + key.setData(origin); + } + + if (curi!=null) { + // success + break; + } + + if (tries>3) { + LOGGER.severe("no item where expected in queue "+classKey); + break; + } + tries++; + LOGGER.severe("Trying get #" + Integer.toString(tries) + + " in queue " + classKey + " with " + getCount() + + " items using key " + + getPrefixClassKey(key.getData())); + } + + return curi; + } + + protected void insertItem(final WorkQueueFrontier frontier, + final CrawlURI curi, boolean overwriteIfPresent) throws IOException { + try { + final BdbMultipleWorkQueues queues = ((BdbFrontier) frontier) + .getWorkQueues(); + queues.put(curi, overwriteIfPresent); + if (LOGGER.isLoggable(Level.FINE)) { + LOGGER.fine("Inserted into " + getPrefixClassKey(this.origin) + + " (count " + Long.toString(getCount())+ "): " + + curi.toString()); + } + } catch (DatabaseException e) { + throw IoUtils.wrapAsIOException(e); + } + } + + /** + * @param byteArray Byte array to get hex string of. + * @return Hex string of passed in byte array (Used logging + * key-prefixes). + */ + protected static String getPrefixClassKey(final byte [] byteArray) { + int zeroIndex = 0; + while(byteArray[zeroIndex]!=0) { + zeroIndex++; + } + try { + return new String(byteArray,0,zeroIndex,"UTF-8"); + } catch (UnsupportedEncodingException e) { + // should be impossible; UTF-8 always available + e.printStackTrace(); + return e.getMessage(); + } + } +} \ No newline at end of file diff --git a/engine/src/main/java/org/archive/crawler/frontier/BucketQueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/BucketQueueAssignmentPolicy.java new file mode 100644 index 00000000..08ba4a43 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/BucketQueueAssignmentPolicy.java @@ -0,0 +1,81 @@ +/* BucketQueueAssignmentPolicy + * + * $Header$ + * + * Created on May 06, 2005 + * + * Copyright (C) 2005 Christian Kohlschuetter + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ +package org.archive.crawler.frontier; + +import org.apache.commons.httpclient.URIException; +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.net.CrawlHost; +import org.archive.modules.net.ServerCache; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Uses the target IPs as basis for queue-assignment, + * distributing them over a fixed number of sub-queues. + * + * @author Christian Kohlschuetter + */ +public class BucketQueueAssignmentPolicy extends QueueAssignmentPolicy { + + private static final long serialVersionUID = 3L; + + private static final int DEFAULT_NOIP_BITMASK = 1023; + private static final int DEFAULT_QUEUES_HOSTS_MODULO = 1021; + + protected ServerCache serverCache; + public ServerCache getServerCache() { + return this.serverCache; + } + @Autowired + public void setServerCache(ServerCache serverCache) { + this.serverCache = serverCache; + } + + public String getClassKey(final CrawlURI curi) { + + CrawlHost host; + try { + host = serverCache.getHostFor(curi.getUURI().getReferencedHost()); + } catch (URIException e) { + // FIXME error handling + e.printStackTrace(); + host = null; + } + if(host == null) { + return "NO-HOST"; + } else if(host.getIP() == null) { + return "NO-IP-".concat(Integer.toString(Math.abs(host.getHostName() + .hashCode()) + & DEFAULT_NOIP_BITMASK)); + } else { + return Integer.toString(Math.abs(host.getIP().hashCode()) + % DEFAULT_QUEUES_HOSTS_MODULO); + } + } + + public int maximumNumberOfKeys() { + return DEFAULT_NOIP_BITMASK + DEFAULT_QUEUES_HOSTS_MODULO + 2; + } +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/CostAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/CostAssignmentPolicy.java new file mode 100644 index 00000000..2a393c47 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/CostAssignmentPolicy.java @@ -0,0 +1,40 @@ +/* CostAssignmentPolicy +* +* $Id$ +* +* Created on Dec 8, 2004 +* +* Copyright (C) 2004 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.crawler.frontier; + +import java.io.Serializable; + +import org.archive.crawler.datamodel.CrawlURI; + +/** + * Calculate a integer 'cost' value for the given CrawlURI. + * + * @author gojomo + */ +public abstract class CostAssignmentPolicy implements Serializable { + + public abstract int costOf(CrawlURI curi); + +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/FrontierJournal.java b/engine/src/main/java/org/archive/crawler/frontier/FrontierJournal.java new file mode 100644 index 00000000..9a3618ae --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/FrontierJournal.java @@ -0,0 +1,356 @@ +/* RecoveryJournal + * + * $Id$ + * + * Created on Jul 20, 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.crawler.frontier; + +import java.io.BufferedReader; +import java.io.EOFException; +import java.io.File; +import java.io.IOException; +import java.util.concurrent.CountDownLatch; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.httpclient.URIException; +import org.archive.checkpointing.Checkpointable; +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.crawler.framework.CrawlController; +import org.archive.crawler.framework.Frontier; +import org.archive.io.CrawlerJournal; +import org.archive.modules.deciderules.DecideRule; +import org.archive.net.UURI; +import org.archive.net.UURIFactory; +import org.json.JSONObject; + +/** + * Helper class for managing a simple Frontier change-events journal which is + * useful for recovering from crawl problems. + * + * By replaying the journal into a new Frontier, its state (at least with + * respect to URIs alreadyIncluded and in pending queues) will match that of the + * original Frontier, allowing a pseudo-resume of a previous crawl, at least as + * far as URI visitation/coverage is concerned. + * + * @author gojomo + */ +public class FrontierJournal extends CrawlerJournal implements Checkpointable { + private static final Logger LOGGER = Logger.getLogger( + FrontierJournal.class.getName()); + + public static final String LOGNAME_RECOVER = "recover.gz"; + + public final static String F_ADD = "F+ "; + public final static String F_EMIT = "Fe "; + public final static String F_DISREGARD = "Fd "; + public final static String F_RESCHEDULE = "Fr "; + public final static String F_SUCCESS = "Fs "; + public final static String F_FAILURE = "Ff "; + + // show recovery progress every this many lines + private final static int PROGRESS_INTERVAL = 1000000; + + // once this many URIs are queued during recovery, allow + // crawl to begin, while enqueuing of other URIs from log + // continues in background + private static final long ENOUGH_TO_START_CRAWLING = 100000; + + /** + * Create a new recovery journal at the given location + * + * @param path Directory to make the recovery journal in. + * @param filename Name to use for recovery journal file. + * @throws IOException + */ + public FrontierJournal(String path, String filename) + throws IOException { + super(path,filename); + timestamp_interval = 10000; + } + + + public synchronized void added(CrawlURI curi) { + accumulatingBuffer.length(0); + this.accumulatingBuffer.append(F_ADD). + append(curi.toString()). + append(" "). + append(curi.getPathFromSeed()). + append(" "). + append(curi.flattenVia()); + writeLine(accumulatingBuffer); + } + + public void finishedSuccess(CrawlURI curi) { + finishedSuccess(curi.toString()); + } + + public void finishedSuccess(UURI uuri) { + finishedSuccess(uuri.toString()); + } + + protected void finishedSuccess(String uuri) { + writeLine(F_SUCCESS, uuri); + } + + public void emitted(CrawlURI curi) { + writeLine(F_EMIT, curi.toString()); + + } + + public void finishedFailure(CrawlURI curi) { + finishedFailure(curi.toString()); + } + + public void finishedDisregard(CrawlURI curi) { + writeLine(F_DISREGARD, curi.toString()); + } + + public void finishedFailure(UURI uuri) { + finishedFailure(uuri.toString()); + } + + public void finishedFailure(String u) { + writeLine(F_FAILURE, u); + } + + public void rescheduled(CrawlURI curi) { + writeLine(F_RESCHEDULE, curi.toString()); + } + + + /** + * Utility method for scanning a recovery journal and applying it to + * a Frontier. + * + * @param params JSONObject of import parameters; see Frontier.importURIS() + * @param controller CrawlController of crawl to update + * @throws IOException + * + * @see org.archive.crawler.framework.Frontier#importURIs(String) + */ + public static void importRecoverLog(final JSONObject params, final CrawlController controller) + throws IOException { + String path = params.optString("path"); + if (path == null) { + throw new IllegalArgumentException("Passed source file is null."); + } + final File source = new File(path); + LOGGER.info("recovering frontier completion state from "+source); + + // first, fill alreadyIncluded with successes (and possibly failures), + // and count the total lines + final int lines = + importCompletionInfoFromLog(source, controller, params); + + LOGGER.info("finished completion state; recovering queues from " + + source); + + // now, re-add anything that was in old frontier and not already + // registered as finished. Do this in a separate thread that signals + // this thread once ENOUGH_TO_START_CRAWLING URIs have been queued. + final CountDownLatch recoveredEnough = new CountDownLatch(1); + new Thread(new Runnable() { + public void run() { + importQueuesFromLog(source, controller, params, lines, + recoveredEnough); + } + }, "queuesRecoveryThread").start(); + + try { + // wait until at least ENOUGH_TO_START_CRAWLING URIs queued + recoveredEnough.await(); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + /** + * Import just the SUCCESS (and possibly FAILURE) URIs from the given + * recovery log into the frontier as considered included. + * + * @param source recovery log file to use + * @param controller CrawlController of crawl to update + * @param retainFailures whether failure ('Ff') URIs should count as done + * @return number of lines in recovery log (for reference) + * @throws IOException + */ + private static int importCompletionInfoFromLog(File source, + CrawlController controller, JSONObject params) throws IOException { + // Scan log for 'Fs' (+maybe 'Ff') lines: add as 'alreadyIncluded' + boolean includeSuccesses = !params.isNull("includeSuccesses"); + boolean includeFailures = !params.isNull("includeFailures"); + boolean includeScheduleds = !params.isNull("includeScheduleds"); + boolean scopeIncludes = !params.isNull("scopeIncludes"); + + Frontier frontier = controller.getFrontier(); + DecideRule scope = (scopeIncludes) ? frontier.getScope() : null; + FrontierJournal newJournal = frontier.getFrontierJournal(); + + BufferedReader br = getBufferedReader(source); + String read; + int lines = 0; + try { + while ((read = br.readLine())!=null) { + lines++; + if(read.length()<4) { + continue; + } + String lineType = read.substring(0, 3); + if(includeSuccesses && F_SUCCESS.equals(lineType) + || includeFailures && F_FAILURE.equals(lineType) + || includeScheduleds && F_ADD.equals(lineType)) { + String s = read.subSequence(3,read.length()).toString(); + try { + UURI u = UURIFactory.getInstance(s); + if(scope!=null) { + CrawlURI caUri = CrawlURI.fromHopsViaString(read.substring(3)); + //TODO:SPRINGY +/// caUri.setStateProvider(controller.getSheetManager()); + // skip out-of-scope URIs if so configured + if(!scope.accepts(caUri)) { + continue; + } + } + frontier.considerIncluded(u); + if (newJournal != null) { + newJournal.writeLine(lineType, u.toString()); + } + } catch (URIException e) { + e.printStackTrace(); + } + } + if((lines%PROGRESS_INTERVAL)==0) { + // every 1 million lines, print progress + LOGGER.info( + "at line " + lines + + " alreadyIncluded count = " + + frontier.discoveredUriCount()); + } + } + } catch (EOFException e) { + // expected in some uncleanly-closed recovery logs; ignore + } finally { + br.close(); + } + return lines; + } + + /** + * Import all ADDs from given recovery log into the frontier's queues + * (excepting those the frontier drops as already having been included) + * + * @param source recovery log file to use + * @param controller CrawlController of crawl to update + * @param params Map of options to apply + * @param enough latch signalling 'enough' URIs queued to begin crawling + */ + private static void importQueuesFromLog(File source, CrawlController controller, + JSONObject params, int lines, CountDownLatch enough) { + BufferedReader br; + String read; + Frontier frontier = controller.getFrontier(); + long queuedAtStart = frontier.queuedUriCount(); + long queuedDuringRecovery = 0; + int qLines = 0; + + boolean scheduleSuccesses = !params.isNull("scheduleSuccesses"); + boolean scheduleFailures = !params.isNull("scheduleFailures"); + boolean scheduleScheduleds = !params.isNull("scheduleScheduleds"); + boolean scopeScheduleds = !params.isNull("scopeScheduleds"); + boolean forceRevisit = !params.isNull("forceRevisit"); + + DecideRule scope = (scopeScheduleds) ? frontier.getScope() : null; + + try { + // Scan log for all 'F+' lines: if not alreadyIncluded, schedule for + // visitation + br = getBufferedReader(source); + try { + while ((read = br.readLine())!=null) { + qLines++; + if(read.length()<4) { + continue; + } + String lineType = read.substring(0, 3); + if(scheduleSuccesses && F_SUCCESS.equals(lineType) + || scheduleFailures && F_FAILURE.equals(lineType) + || scheduleScheduleds && F_ADD.equals(lineType)) { + + try { + CrawlURI caUri = CrawlURI.fromHopsViaString(read.substring(3)); + + //TODO:SPRINGY +// caUri.setStateProvider(controller.getSheetManager()); + if(scope!=null) { + // skip out-of-scope URIs if so configured + if(!scope.accepts(caUri)) { + continue; + } + } + + caUri.setForceFetch(forceRevisit); + + frontier.schedule(caUri); + + queuedDuringRecovery = + frontier.queuedUriCount() - queuedAtStart; + if(((queuedDuringRecovery + 1) % + ENOUGH_TO_START_CRAWLING) == 0) { + enough.countDown(); + } + } catch (URIException e) { + LOGGER.log(Level.WARNING, "bad URI during " + + "log-recovery of queue contents ",e); + // and continue... + } catch (RuntimeException e) { + LOGGER.log(Level.SEVERE, "exception during " + + "log-recovery of queue contents ",e); + // and continue, though this may be risky + // if the exception wasn't a trivial NPE + // or wrapped interrupted-exception... + } + } + if((qLines%PROGRESS_INTERVAL)==0) { + // every 1 million lines, print progress + LOGGER.info( + "through line " + + qLines + "/" + lines + + " queued count = " + + frontier.queuedUriCount()); + } + } + } catch (EOFException e) { + // no problem: untidy end of recovery journal + } finally { + br.close(); + } + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + LOGGER.info("finished recovering frontier from "+source+" " + +qLines+" lines processed"); + enough.countDown(); + } +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/HostnameQueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/HostnameQueueAssignmentPolicy.java new file mode 100644 index 00000000..c3fd7ce7 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/HostnameQueueAssignmentPolicy.java @@ -0,0 +1,96 @@ +/* HostnameQueueAssignmentPolicy +* +* $Id$ +* +* Created on Oct 5, 2004 +* +* Copyright (C) 2004 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.crawler.frontier; + +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.httpclient.URIException; +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.net.UURI; +import org.archive.net.UURIFactory; + +/** + * QueueAssignmentPolicy based on the hostname:port evident in the given + * CrawlURI. + * + * @author gojomo + */ +public class HostnameQueueAssignmentPolicy extends QueueAssignmentPolicy { + + private static final long serialVersionUID = 3L; + + private static final Logger logger = Logger + .getLogger(HostnameQueueAssignmentPolicy.class.getName()); + /** + * When neat host-based class-key fails us + */ + private static String DEFAULT_CLASS_KEY = "default..."; + + private static final String DNS = "dns"; + + public String getClassKey(CrawlURI cauri) { + String scheme = cauri.getUURI().getScheme(); + String candidate = null; + try { + if (scheme.equals(DNS)){ + if (cauri.getVia() != null) { + // Special handling for DNS: treat as being + // of the same class as the triggering URI. + // When a URI includes a port, this ensures + // the DNS lookup goes atop the host:port + // queue that triggered it, rather than + // some other host queue + UURI viaUuri = UURIFactory.getInstance(cauri.flattenVia()); + candidate = viaUuri.getAuthorityMinusUserinfo(); + // adopt scheme of triggering URI + scheme = viaUuri.getScheme(); + } else { + candidate= cauri.getUURI().getReferencedHost(); + } + } else { + candidate = cauri.getUURI().getAuthorityMinusUserinfo(); + } + + if(candidate == null || candidate.length() == 0) { + candidate = DEFAULT_CLASS_KEY; + } + } catch (URIException e) { + logger.log(Level.INFO, + "unable to extract class key; using default", e); + candidate = DEFAULT_CLASS_KEY; + } + if (scheme != null && scheme.equals(UURIFactory.HTTPS)) { + // If https and no port specified, add default https port to + // distinguish https from http server without a port. + if (!candidate.matches(".+:[0-9]+")) { + candidate += UURIFactory.HTTPS_PORT; + } + } + // Ensure classKeys are safe as filenames on NTFS + return candidate.replace(':','#'); + } + +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/IPQueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/IPQueueAssignmentPolicy.java new file mode 100644 index 00000000..30c3f90f --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/IPQueueAssignmentPolicy.java @@ -0,0 +1,63 @@ +/* IPQueueAssignmentPolicy +* +* $Id$ +* +* Created on Oct 5, 2004 +* +* Copyright (C) 2004 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.crawler.frontier; + +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.net.CrawlHost; +import org.archive.modules.net.ServerCache; +import org.archive.modules.net.ServerCacheUtil; +import org.springframework.beans.factory.annotation.Autowired; + + +/** + * Uses target IP as basis for queue-assignment, unless it is unavailable, + * in which case it behaves as HostnameQueueAssignmentPolicy. + * + * @author gojomo + */ +public class IPQueueAssignmentPolicy + extends HostnameQueueAssignmentPolicy { + private static final long serialVersionUID = 3L; + + protected ServerCache serverCache; + public ServerCache getServerCache() { + return this.serverCache; + } + @Autowired + public void setServerCache(ServerCache serverCache) { + this.serverCache = serverCache; + } + + public String getClassKey(CrawlURI cauri) { + CrawlHost host = ServerCacheUtil.getHostFor(serverCache, + cauri.getUURI()); + if (host == null || host.getIP() == null) { + // if no server or no IP, use superclass implementation + return super.getClassKey(cauri); + } + // use dotted-decimal IP address + return host.getIP().getHostAddress(); + } +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/QueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/QueueAssignmentPolicy.java new file mode 100644 index 00000000..2afe04ab --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/QueueAssignmentPolicy.java @@ -0,0 +1,60 @@ +/* QueueAssignmentPolicy +* +* $Id$ +* +* Created on Oct 5, 2004 +* +* Copyright (C) 2004 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.crawler.frontier; + +import java.io.Serializable; + +import org.archive.crawler.datamodel.CrawlURI; + +/** + * Establishes a mapping from CrawlURIs to String keys (queue names). + * + * @author gojomo + */ +public abstract class QueueAssignmentPolicy implements Serializable { + /** + * Get the String key (name) of the queue to which the + * CrawlURI should be assigned. + * + * Note that changes to the CrawlURI, or its associated + * components (such as CrawlServer), may change its queue + * assignment. + * @param controller This crawls' controller. + * + * @param cauri CandidateURI to calculate class key for. + * @return the String key of the queue to assign the CrawlURI + */ + public abstract String getClassKey(CrawlURI cauri); + + /** + * Returns the maximum number of different keys this policy + * can create. If there is no maximum, -1 is returned (default). + * + * @return Maximum number of different keys, or -1 if unbounded. + */ + public int maximumNumberOfKeys() { + return -1; + } +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/RecyclingSerialBinding.java b/engine/src/main/java/org/archive/crawler/frontier/RecyclingSerialBinding.java new file mode 100644 index 00000000..d1371b0a --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/RecyclingSerialBinding.java @@ -0,0 +1,112 @@ +/* RecyclingSerialBinding +* +* $Id$ +* +* Created on May 25, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.crawler.frontier; + +import java.io.IOException; + +import com.sleepycat.bind.serial.ClassCatalog; +import com.sleepycat.bind.serial.SerialBinding; +import com.sleepycat.bind.serial.SerialOutput; +import com.sleepycat.je.DatabaseEntry; +import com.sleepycat.util.FastOutputStream; +import com.sleepycat.util.RuntimeExceptionWrapper; + +/** + * A SerialBinding that recycles a single FastOutputStream per + * thread, avoiding reallocation of the internal buffer for + * either repeated serializations or because of mid-serialization + * expansions. (Cached stream's buffer will quickly grow to a size + * sufficient for all serialized instances.) + * + * @author gojomo + */ +public class RecyclingSerialBinding extends SerialBinding { + /** + * Thread-local cache of reusable FastOutputStream + */ + ThreadLocal fastOutputStreamHolder + = new ThreadLocal(); + + private ClassCatalog classCatalog; + private Class baseClass; + + /** + * Constructor. Save parameters locally, as superclass + * fields are private. + * + * @param classCatalog is the catalog to hold shared class information + * + * @param baseClass is the base class for serialized objects stored using + * this binding + */ + @SuppressWarnings("unchecked") + public RecyclingSerialBinding(ClassCatalog classCatalog, Class baseClass) { + super(classCatalog, baseClass); + this.classCatalog = classCatalog; + this.baseClass = baseClass; + } + + /** + * Copies superclass simply to allow different source for FastOoutputStream. + * + * @see com.sleepycat.bind.serial.SerialBinding#entryToObject + */ + public void objectToEntry(Object object, DatabaseEntry entry) { + + if (baseClass != null && !baseClass.isInstance(object)) { + throw new IllegalArgumentException( + "Data object class (" + object.getClass() + + ") not an instance of binding's base class (" + + baseClass + ')'); + } + FastOutputStream fo = getFastOutputStream(); + try { + SerialOutput jos = new SerialOutput(fo, classCatalog); + jos.writeObject(object); + } catch (IOException e) { + throw new RuntimeExceptionWrapper(e); + } + + byte[] hdr = SerialOutput.getStreamHeader(); + entry.setData(fo.getBufferBytes(), hdr.length, + fo.getBufferLength() - hdr.length); + } + + /** + * Get the cached (and likely pre-grown to efficient size) FastOutputStream, + * creating it if necessary. + * + * @return FastOutputStream + */ + private FastOutputStream getFastOutputStream() { + FastOutputStream fo = (FastOutputStream) fastOutputStreamHolder.get(); + if (fo == null) { + fo = new FastOutputStream(); + fastOutputStreamHolder.set(fo); + } + fo.reset(); + return fo; + } +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/SurtAuthorityQueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/SurtAuthorityQueueAssignmentPolicy.java new file mode 100644 index 00000000..1e2243a1 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/SurtAuthorityQueueAssignmentPolicy.java @@ -0,0 +1,99 @@ +/* SurtAuthorityQueueAssignmentPolicy +* +* $Id$ +* +* Created on Oct 5, 2004 +* +* Copyright (C) 2004 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.crawler.frontier; + +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.httpclient.URIException; +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.net.UURI; +import org.archive.net.UURIFactory; + +/** + * SurtAuthorityQueueAssignmentPolicy based on the surt form of hostname. + */ +public class SurtAuthorityQueueAssignmentPolicy +extends QueueAssignmentPolicy { + + private static final long serialVersionUID = 3L; + + private static final Logger logger = Logger + .getLogger(SurtAuthorityQueueAssignmentPolicy.class.getName()); + + /** + * When neat host-based class-key fails us + */ + private static String DEFAULT_CLASS_KEY = "default..."; + + private static final String DNS = "dns"; + + public String getClassKey(CrawlURI cauri) { + String scheme = cauri.getUURI().getScheme(); + String candidate = null; + try { + if (scheme.equals(DNS)) { + UURI effectiveuuri; + if (cauri.getVia() != null) { + // Special handling for DNS: treat as being + // of the same class as the triggering URI. + // When a URI includes a port, this ensures + // the DNS lookup goes atop the host:port + // queue that triggered it, rather than + // some other host queue + effectiveuuri = UURIFactory.getInstance(cauri.flattenVia()); + } else { + // To get the dns surt form, create a fake http version + // (Gordon suggestion). + effectiveuuri = UURIFactory.getInstance("http://" + + cauri.getUURI().getPath()); + } + candidate = getSurtAuthority(effectiveuuri.getSurtForm()); + } else { + candidate = getSurtAuthority(cauri.getUURI().getSurtForm()); + } + + if(candidate == null || candidate.length() == 0) { + candidate = DEFAULT_CLASS_KEY; + } + } catch (URIException e) { + logger.log(Level.INFO, + "unable to extract class key; using default", e); + candidate = DEFAULT_CLASS_KEY; + } + // Ensure classKeys are safe as filenames on NTFS + return candidate.replace(':','#'); + } + + protected String getSurtAuthority(String surt) { + int indexOfOpen = surt.indexOf("://("); + int indexOfClose = surt.indexOf(")"); + if (indexOfOpen == -1 || indexOfClose == -1 + || ((indexOfOpen + 4) >= indexOfClose)) { + return DEFAULT_CLASS_KEY; + } + return surt.substring(indexOfOpen + 4, indexOfClose); + } +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/TopmostAssignedSurtQueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/TopmostAssignedSurtQueueAssignmentPolicy.java new file mode 100644 index 00000000..da30f77d --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/TopmostAssignedSurtQueueAssignmentPolicy.java @@ -0,0 +1,47 @@ +/* AbstractFrontier + * + * $Id: AbstractFrontier.java 5053 2007-04-10 02:34:20Z gojomo $ + * + * Created on June 18, 2007 + * + * Copyright (C) 2007 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.crawler.frontier; + +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.net.PublicSuffixes; + +/** + * Create a queueKey based on the SURT authority, reduced to the + * public-suffix-plus-one domain (topmost assignable domain). + * + * @author gojomo + */ +public class TopmostAssignedSurtQueueAssignmentPolicy extends + SurtAuthorityQueueAssignmentPolicy { + private static final long serialVersionUID = -1533545293624791702L; + + @Override + public String getClassKey(CrawlURI cauri) { + String candidate = super.getClassKey(cauri); + candidate = PublicSuffixes.reduceSurtToTopmostAssigned(candidate); + return candidate; + } + +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/UnitCostAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/UnitCostAssignmentPolicy.java new file mode 100644 index 00000000..ec341124 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/UnitCostAssignmentPolicy.java @@ -0,0 +1,44 @@ +/* UnitCostAssignmentPolicy +* +* $Id$ +* +* Created on Dec 8, 2004 +* +* Copyright (C) 2004 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.crawler.frontier; + +import org.archive.crawler.datamodel.CrawlURI; + +/** + * A CostAssignment policy that uses a constant value of 1 for all CrawlURIs. + * + * @author gojomo + */ +public class UnitCostAssignmentPolicy extends CostAssignmentPolicy { + private static final long serialVersionUID = 1L; + + /* (non-Javadoc) + * @see org.archive.crawler.frontier.CostAssignmentPolicy#costOf(org.archive.crawler.datamodel.CrawlURI) + */ + public int costOf(CrawlURI curi) { + return 1; + } + +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/WagCostAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/WagCostAssignmentPolicy.java new file mode 100644 index 00000000..d560f42d --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/WagCostAssignmentPolicy.java @@ -0,0 +1,71 @@ +/* WagCostAssignmentPolicy +* +* $Id$ +* +* Created on Dec 10, 2004 +* +* Copyright (C) 2004 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.crawler.frontier; + +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.net.UURI; + +/** + * A CostAssignmentPolicy based on some wild guesses of kinds of URIs + * that should be deferred into the (potentially never-crawled) future. + * + * @author gojomo + */ +public class WagCostAssignmentPolicy extends CostAssignmentPolicy { + private static final long serialVersionUID = 1L; + + /** + * Add constant penalties for certain features of URI (and + * its 'via') that make it more delayable/skippable. + * + * @param curi CrawlURI to be assigned a cost + * + * @see org.archive.crawler.frontier.CostAssignmentPolicy#costOf(org.archive.crawler.datamodel.CrawlURI) + */ + public int costOf(CrawlURI curi) { + int cost = 1; + UURI uuri = curi.getUURI(); + if (uuri.hasQuery()) { + // has query string + cost++; + int qIndex = uuri.toString().indexOf('?'); + if (curi.flattenVia().startsWith(uuri.toString().substring(0,qIndex))) { + // non-query-string portion of URI is same as previous + cost++; + } + // TODO: other potential query-related cost penalties: + // - more than X query-string attributes + // - calendarish terms + // - query-string over certain size + } + // TODO: other potential path-based penalties + // - new path is simply extension of via path + // - many path segments + // TODO: other potential hops-based penalties + // - more than X hops + // - each speculative hop + return cost; + } +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java b/engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java new file mode 100644 index 00000000..bd17ad43 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java @@ -0,0 +1,678 @@ +package org.archive.crawler.frontier; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.Serializable; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.Delayed; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.crawler.framework.Frontier; +import org.archive.crawler.frontier.precedence.PrecedenceProvider; +import org.archive.crawler.frontier.precedence.SimplePrecedenceProvider; +import org.archive.modules.ProcessorURI; +import org.archive.modules.fetcher.FetchStats; +import org.archive.modules.fetcher.FetchStats.Stage; +import org.archive.util.ArchiveUtils; +import org.archive.util.MultiReporter; + +/** + * A single queue of related URIs to visit, grouped by a classKey + * (typically "hostname:port" or similar) + * + * @author gojomo + * @author Christian Kohlschuetter + */ +public abstract class WorkQueue implements Frontier.FrontierGroup, + Serializable, MultiReporter, Delayed { + private static final long serialVersionUID = -3199666138837266341L; + private static final Logger logger = + Logger.getLogger(WorkQueue.class.getName()); + + /** The classKey */ + protected final String classKey; + + /** whether queue is active (ready/in-process/snoozed) or on a waiting queue */ + private boolean active = true; + + /** Total number of stored items */ + private long count = 0; + + /** Total number of items ever enqueued */ + private long enqueueCount = 0; + + /** Whether queue is already in lifecycle stage */ + private boolean isHeld = false; + + /** Time to wake, if snoozed */ + private long wakeTime = 0; + + /** assigned precedence */ + private PrecedenceProvider precedenceProvider = new SimplePrecedenceProvider(1); + + /** set of by-precedence inactive-queues on which WorkQueue is waiting */ + private Set onInactiveQueues = new HashSet(); + + /** Running 'budget' indicating whether queue should stay active */ + private int sessionBalance = 0; + + /** Cost of the last item to be charged against queue */ + private int lastCost = 0; + + /** Total number of items charged against queue; with totalExpenditure + * can be used to calculate 'average cost'. */ + private long costCount = 0; + + /** Running tally of total expenditures on this queue */ + private long totalExpenditure = 0; + + /** Total to spend on this queue over its lifetime */ + private long totalBudget = 0; + + /** The next item to be returned */ + protected CrawlURI peekItem = null; + + /** Last URI enqueued */ + private String lastQueued; + + /** Last URI peeked */ + private String lastPeeked; + + /** time of last dequeue (disposition of some URI) **/ + private long lastDequeueTime; + + /** count of errors encountered */ + private long errorCount = 0; + + /** Substats for all CrawlURIs in this group */ + protected FetchStats substats = new FetchStats(); + + private boolean retired; + + public WorkQueue(final String pClassKey) { + this.classKey = pClassKey; + } + + /** + * Delete URIs matching the given pattern from this queue. + * @param frontier + * @param match + * @return count of deleted URIs + */ + public long deleteMatching(final WorkQueueFrontier frontier, String match) { + try { + final long deleteCount = deleteMatchingFromQueue(frontier, match); + this.count -= deleteCount; + return deleteCount; + } catch (IOException e) { + //FIXME better exception handling + e.printStackTrace(); + throw new RuntimeException(e); + } + } + + /** + * Add the given CrawlURI, noting its addition in running count. (It + * should not already be present.) + * + * @param frontier Work queues manager. + * @param curi CrawlURI to insert. + */ + protected void enqueue(final WorkQueueFrontier frontier, + CrawlURI curi) { + try { + insert(frontier, curi, false); + } catch (IOException e) { + //FIXME better exception handling + e.printStackTrace(); + throw new RuntimeException(e); + } + count++; + enqueueCount++; + } + + /** + * Return the topmost queue item -- and remember it, + * such that even later higher-priority inserts don't + * change it. + * + * TODO: evaluate if this is really necessary + * @param frontier Work queues manager + * + * @return topmost queue item, or null + */ + public CrawlURI peek(final WorkQueueFrontier frontier) { + if(peekItem == null && count > 0) { + try { + peekItem = peekItem(frontier); + } catch (IOException e) { + //FIXME better exception handling + logger.log(Level.SEVERE,"peek failure",e); + e.printStackTrace(); + // throw new RuntimeException(e); + } + if(peekItem != null) { + lastPeeked = peekItem.toString(); + } + } + return peekItem; + } + + /** + * Remove the peekItem from the queue and adjusts the count. + * + * @param frontier Work queues manager. + */ + protected void dequeue(final WorkQueueFrontier frontier, CrawlURI expected) { + try { + deleteItem(frontier, peekItem); + } catch (IOException e) { + //FIXME better exception handling + e.printStackTrace(); + throw new RuntimeException(e); + } + unpeek(expected); + count--; + lastDequeueTime = System.currentTimeMillis(); + } + + /** + * Set the session 'activity budget balance' to the given value + * + * @param balance to use + */ + public void setSessionBalance(int balance) { + this.sessionBalance = balance; + } + + /** + * Return current session 'activity budget balance' + * + * @return session balance + */ + public int getSessionBalance() { + return this.sessionBalance; + } + + /** + * Set the total expenditure level allowable before queue is + * considered inherently 'over-budget'. + * + * @param budget + */ + public void setTotalBudget(long budget) { + this.totalBudget = budget; + } + + /** + * Check whether queue has temporarily or permanently exceeded + * its budget. + * + * @return true if queue is over its set budget(s) + */ + public boolean isOverBudget() { + // check whether running balance is depleted + // or totalExpenditure exceeds totalBudget + return this.sessionBalance <= 0 + || (this.totalBudget >= 0 && this.totalExpenditure >= this.totalBudget); + } + + /** + * Return the tally of all expenditures on this queue + * + * @return total amount expended on this queue + */ + public long getTotalExpenditure() { + return totalExpenditure; + } + + /** + * Increase the internal running budget to be used before + * deactivating the queue + * + * @param amount amount to increment + * @return updated budget value + */ + public int incrementSessionBalance(int amount) { + this.sessionBalance = this.sessionBalance + amount; + return this.sessionBalance; + } + + /** + * Decrease the internal running budget by the given amount. + * @param amount tp decrement + * @return updated budget value + */ + public int expend(int amount) { + this.sessionBalance = this.sessionBalance - amount; + this.totalExpenditure = this.totalExpenditure + amount; + this.lastCost = amount; + this.costCount++; + return this.sessionBalance; + } + + /** + * A URI should not have been charged against queue (eg + * it was disregarded); return the amount expended + * @param amount to return + * @return updated budget value + */ + public int refund(int amount) { + this.sessionBalance = this.sessionBalance + amount; + this.totalExpenditure = this.totalExpenditure - amount; + this.costCount--; + return this.sessionBalance; + } + + /** + * Note an error and assess an extra penalty. + * @param penalty additional amount to deduct + */ + public void noteError(int penalty) { + this.sessionBalance = this.sessionBalance - penalty; + this.totalExpenditure = this.totalExpenditure + penalty; + errorCount++; + } + + /** + * @param l + */ + public void setWakeTime(long l) { + wakeTime = l; + } + + /** + * @return wakeTime + */ + public long getWakeTime() { + return wakeTime; + } + + /** + * @return classKey, the 'identifier', for this queue. + */ + public String getClassKey() { + return this.classKey; + } + + /** + * Clear isHeld to false + */ + public void clearHeld() { + isHeld = false; + if(logger.isLoggable(Level.FINE)) { + logger.log(Level.FINE, + "queue unheld: " + getClassKey()); + } + } + + /** + * Whether the queue is already in a lifecycle stage -- + * such as ready, in-progress, snoozed -- and thus should + * not be redundantly inserted to readyClassQueues + * + * @return isHeld + */ + public boolean isHeld() { + return isHeld; + } + + /** + * Set isHeld to true + */ + public void setHeld() { + isHeld = true; + if(logger.isLoggable(Level.FINE)) { + logger.log(Level.FINE, + "queue held: " + getClassKey()); + } + } + + /** + * Forgive the peek, allowing a subsequent peek to + * return a different item. + * + */ + public void unpeek(CrawlURI expected) { + assert expected == peekItem : "unexpected peekItem"; + peekItem = null; + } + + /* (non-Javadoc) + * @see java.util.concurrent.Delayed#getDelay(java.util.concurrent.TimeUnit) + */ + public long getDelay(TimeUnit unit) { + return unit.convert( + getWakeTime()-System.currentTimeMillis(), + TimeUnit.MILLISECONDS); + } + + public final int compareTo(Delayed obj) { + if(this == obj) { + return 0; // for exact identity only + } + WorkQueue other = (WorkQueue) obj; + if(getWakeTime() > other.getWakeTime()) { + return 1; + } + if(getWakeTime() < other.getWakeTime()) { + return -1; + } + // at this point, the ordering is arbitrary, but still + // must be consistent/stable over time + return this.classKey.compareTo(other.getClassKey()); + } + + /** + * Update the given CrawlURI, which should already be present. (This + * is not checked.) Equivalent to an enqueue without affecting the count. + * + * @param frontier Work queues manager. + * @param curi CrawlURI to update. + */ + public void update(final WorkQueueFrontier frontier, CrawlURI curi) { + try { + insert(frontier, curi, true); + } catch (IOException e) { + //FIXME better exception handling + e.printStackTrace(); + throw new RuntimeException(e); + } + } + + /** + * Count of URIs in this queue. Only precise if called within frontier's + * manager thread. + * + * @return Returns the count. + */ + public long getCount() { + return this.count; + } + + /** + * Insert the given curi, whether it is already present or not. + * @param frontier WorkQueueFrontier. + * @param curi CrawlURI to insert. + * @throws IOException + */ + private void insert(final WorkQueueFrontier frontier, CrawlURI curi, + boolean overwriteIfPresent) + throws IOException { + insertItem(frontier, curi, overwriteIfPresent); + lastQueued = curi.toString(); + } + + /** + * Insert the given curi, whether it is already present or not. + * Hook for subclasses. + * + * @param frontier WorkQueueFrontier. + * @param curi CrawlURI to insert. + * @throws IOException if there was a problem while inserting the item + */ + protected abstract void insertItem(final WorkQueueFrontier frontier, + CrawlURI curi, boolean overwriteIfPresent) throws IOException; + + /** + * Delete URIs matching the given pattern from this queue. + * @param frontier WorkQueues manager. + * @param match the pattern to match + * @return count of deleted URIs + * @throws IOException if there was a problem while deleting + */ + protected abstract long deleteMatchingFromQueue( + final WorkQueueFrontier frontier, final String match) + throws IOException; + + /** + * Removes the given item from the queue. + * + * This is only used to remove the first item in the queue, + * so it is not necessary to implement a random-access queue. + * + * @param frontier Work queues manager. + * @throws IOException if there was a problem while deleting the item + */ + protected abstract void deleteItem(final WorkQueueFrontier frontier, + final CrawlURI item) throws IOException; + + /** + * Returns first item from queue (does not delete) + * + * @return The peeked item, or null + * @throws IOException if there was a problem while peeking + */ + protected abstract CrawlURI peekItem(final WorkQueueFrontier frontier) + throws IOException; + + /** + * Suspends this WorkQueue. Closes all connections to resources etc. + * + * @param frontier + * @throws IOException + */ + protected void suspend(final WorkQueueFrontier frontier) throws IOException { + } + + /** + * Resumes this WorkQueue. Eventually opens connections to resources etc. + * + * @param frontier + * @throws IOException + */ + protected void resume(final WorkQueueFrontier frontier) throws IOException { + } + + public void setActive(final WorkQueueFrontier frontier, final boolean b) { + if(active != b) { + active = b; + if(logger.isLoggable(Level.FINE)) { + logger.log(Level.FINE, + (active ? "queue set active: " : "queue unset active: ") + + this.getClassKey()); + } + try { + if(active) { + resume(frontier); + } else { + suspend(frontier); + } + } catch (IOException e) { + //FIXME better exception handling + e.printStackTrace(); + throw new RuntimeException(e); + } + } + } + + // + // Reporter + // + + /* (non-Javadoc) + * @see org.archive.util.Reporter#getReports() + */ + public String[] getReports() { + return new String[] {}; + } + + /* (non-Javadoc) + * @see org.archive.util.Reporter#reportTo(java.io.Writer) + */ + public void reportTo(PrintWriter writer) { + reportTo(null,writer); + } + + /* (non-Javadoc) + * @see org.archive.util.Reporter#singleLineReportTo(java.io.Writer) + */ + public void singleLineReportTo(PrintWriter writer) { + // queue name + writer.print(classKey); + writer.print(" "); + // precedence + writer.print(getPrecedence()); + writer.print(" "); + // count of items + writer.print(Long.toString(count)); + writer.print(" "); + // enqueue count + writer.print(Long.toString(enqueueCount)); + writer.print(" "); + writer.print(sessionBalance); + writer.print(" "); + writer.print(lastCost); + writer.print("("); + writer.print(ArchiveUtils.doubleToString( + ((double) totalExpenditure / costCount), 1)); + writer.print(")"); + writer.print(" "); + // last dequeue time, if any, or '-' + if (lastDequeueTime != 0) { + writer.print(ArchiveUtils.getLog17Date(lastDequeueTime)); + } else { + writer.print("-"); + } + writer.print(" "); + // wake time if snoozed, or '-' + if (wakeTime != 0) { + writer.print(ArchiveUtils.formatMillisecondsToConventional(wakeTime - System.currentTimeMillis())); + } else { + writer.print("-"); + } + writer.print(" "); + writer.print(Long.toString(totalExpenditure)); + writer.print("/"); + writer.print(Long.toString(totalBudget)); + writer.print(" "); + writer.print(Long.toString(errorCount)); + writer.print(" "); + writer.print(lastPeeked); + writer.print(" "); + writer.print(lastQueued); + writer.print("\n"); + } + + /* (non-Javadoc) + * @see org.archive.util.Reporter#singleLineLegend() + */ + public String singleLineLegend() { + return "queue precedence currentSize totalEnqueues sessionBalance " + + "lastCost (averageCost) lastDequeueTime wakeTime " + + "totalSpend/totalBudget errorCount lastPeekUri lastQueuedUri"; + } + + /* (non-Javadoc) + * @see org.archive.util.Reporter#singleLineReport() + */ + public String singleLineReport() { + return ArchiveUtils.singleLineReport(this); + } + + /** + * @param writer + * @throws IOException + */ + public void reportTo(String name, PrintWriter writer) { + // name is ignored: only one kind of report for now + writer.print("Queue "); + writer.print(classKey); + writer.print(" (p"); + writer.print(getPrecedence()); + writer.print(")\n"); + writer.print(" "); + writer.print(Long.toString(count)); + writer.print(" items"); + if (wakeTime != 0) { + writer.print("\n wakes in: "+ArchiveUtils.formatMillisecondsToConventional(wakeTime - System.currentTimeMillis())); + } + writer.print("\n last enqueued: "); + writer.print(lastQueued); + writer.print("\n last peeked: "); + writer.print(lastPeeked); + writer.print("\n"); + writer.print(" total expended: "); + writer.print(Long.toString(totalExpenditure)); + writer.print(" (total budget: "); + writer.print(Long.toString(totalBudget)); + writer.print(")\n"); + writer.print(" active balance: "); + writer.print(sessionBalance); + writer.print("\n last(avg) cost: "); + writer.print(lastCost); + writer.print("("); + writer.print(ArchiveUtils.doubleToString( + ((double) totalExpenditure / costCount), 1)); + writer.print(")\n "); + writer.print(getSubstats().singleLineLegend()); + writer.print("\n "); + writer.print(getSubstats().singleLineReport()); + writer.print("\n "); + writer.print(getPrecedenceProvider().singleLineLegend()); + writer.print("\n "); + writer.print(getPrecedenceProvider().singleLineReport()); + writer.print("\n\n"); + } + + public FetchStats getSubstats() { + return substats; + } + + /** + * Set the retired status of this queue. + * + * @param b new value for retired status + */ + public void setRetired(boolean b) { + this.retired = b; + } + + public boolean isRetired() { + return retired; + } + + /** + * @return the precedenceProvider + */ + public PrecedenceProvider getPrecedenceProvider() { + return precedenceProvider; + } + + /** + * @param precedenceProvider the precedenceProvider to set + */ + public void setPrecedenceProvider(PrecedenceProvider precedenceProvider) { + this.precedenceProvider = precedenceProvider; + } + + /** + * @return the precedence + */ + public int getPrecedence() { + return precedenceProvider.getPrecedence(); + } + + /** + * @return the onInactiveQueues + */ + public Set getOnInactiveQueues() { + return onInactiveQueues; + } + + /* (non-Javadoc) + * @see org.archive.modules.fetcher.FetchStats.HasFetchStats#tally(org.archive.modules.ProcessorURI, org.archive.modules.fetcher.FetchStats.Stage) + */ + public void tally(ProcessorURI curi, Stage stage) { + substats.tally(curi, stage); + precedenceProvider.tally(curi, stage); + } + + public boolean isActive() { + return active; + } +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java new file mode 100644 index 00000000..4a596178 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java @@ -0,0 +1,1718 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.frontier; + +import static org.archive.crawler.datamodel.CoreAttributeConstants.A_FORCE_RETIRE; +import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.DEFERRED_FOR_RETRY; +import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.DISREGARDED; +import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.FAILED; +import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.SUCCEEDED; +import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED; +import static org.archive.modules.fetcher.FetchStatusCodes.S_RUNTIME_EXCEPTION; + +import java.io.Closeable; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.Serializable; +import java.lang.ref.SoftReference; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Queue; +import java.util.SortedMap; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.DelayQueue; +import java.util.concurrent.Delayed; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Pattern; + +import org.apache.commons.collections.Bag; +import org.apache.commons.collections.BagUtils; +import org.apache.commons.collections.bag.HashBag; +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.crawler.datamodel.UriUniqFilter; +import org.archive.crawler.datamodel.UriUniqFilter.CrawlUriReceiver; +import org.archive.crawler.event.CrawlURIDispositionEvent; +import org.archive.crawler.framework.ToeThread; +import org.archive.crawler.frontier.precedence.BaseQueuePrecedencePolicy; +import org.archive.crawler.frontier.precedence.CostUriPrecedencePolicy; +import org.archive.crawler.frontier.precedence.QueuePrecedencePolicy; +import org.archive.crawler.frontier.precedence.UriPrecedencePolicy; +import org.archive.net.UURI; +import org.archive.spring.KeyedProperties; +import org.archive.util.ArchiveUtils; +import org.archive.util.Transform; +import org.archive.util.Transformer; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.context.support.AbstractApplicationContext; + +import com.sleepycat.je.DatabaseException; + +/** + * A common Frontier base using several queues to hold pending URIs. + * + * Uses in-memory map of all known 'queues' inside a single database. + * Round-robins between all queues. + * + * @author Gordon Mohr + * @author Christian Kohlschuetter + */ +public abstract class WorkQueueFrontier extends AbstractFrontier +implements Closeable, CrawlUriReceiver, Serializable, +ApplicationContextAware { + private static final long serialVersionUID = 570384305871965843L; + + /** truncate reporting of queues at some large but not unbounded number */ + private static final int REPORT_MAX_QUEUES = 2000; + + /** + * If we know that only a small amount of queues is held in memory, + * we can avoid using a disk-based BigMap. + * This only works efficiently if the WorkQueue does not hold its + * entries in memory as well. + */ + private static final int MAX_QUEUES_TO_HOLD_ALLQUEUES_IN_MEMORY = 3000; + + /** + * When a snooze target for a queue is longer than this amount, the queue + * will be "long snoozed" instead of "short snoozed". A "long snoozed" + * queue may be swapped to disk because it's not needed soon. + */ + long snoozeLongMs = 5L*60L*1000L; + public long getSnoozeLongMs() { + return snoozeLongMs; + } + public void setSnoozeLongMs(long snooze) { + this.snoozeLongMs = snooze; + } + + private static final Logger logger = + Logger.getLogger(WorkQueueFrontier.class.getName()); + + // ApplicationContextAware implementation, for eventing + AbstractApplicationContext appCtx; + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + this.appCtx = (AbstractApplicationContext)applicationContext; + } + + /** + * Whether queues should start INACTIVE (only becoming active + * when needed to keep the crawler busy), or if queues should + * start out ready (which means all nonempty queues are + * considered in a round-robin fashion) + * + * @return true if new queues should held inactive + */ + boolean holdQueues = true; + public boolean getHoldQueues() { + return holdQueues; + } + public void setHoldQueues(boolean holdQueues) { + this.holdQueues = holdQueues; + + } + + /** amount to replenish budget on each activation (duty cycle) */ + { + setBalanceReplenishAmount(3000); + } + public int getBalanceReplenishAmount() { + return (Integer) kp.get("balanceReplenishAmount"); + } + public void setBalanceReplenishAmount(int replenish) { + kp.put("balanceReplenishAmount",replenish); + } + + + /** budget penalty for an error fetch */ + { + setErrorPenaltyAmount(100); + } + public int getErrorPenaltyAmount() { + return (Integer) kp.get("errorPenaltyAmount"); + } + public void setErrorPenaltyAmount(int penalty) { + kp.put("errorPenaltyAmount",penalty); + } + + /** total expenditure to allow a queue before 'retiring' it */ + { + setQueueTotalBudget(-1L); + } + public long getQueueTotalBudget() { + return (Long) kp.get("queueTotalBudget"); + } + public void setQueueTotalBudget(long budget) { + kp.put("queueTotalBudget",budget); + } + + /** cost assignment policy to use. */ + { + setCostAssignmentPolicy(new UnitCostAssignmentPolicy()); + } + public CostAssignmentPolicy getCostAssignmentPolicy() { + return (CostAssignmentPolicy) kp.get("costAssignmentPolicy"); + } + public void setCostAssignmentPolicy(CostAssignmentPolicy policy) { + kp.put("costAssignmentPolicy",policy); + } + + /** queue precedence assignment policy to use. */ + { + setQueuePrecedencePolicy(new BaseQueuePrecedencePolicy()); + } + public QueuePrecedencePolicy getQueuePrecedencePolicy() { + return (QueuePrecedencePolicy) kp.get("queuePrecedencePolicy"); + } + public void setQueuePrecedencePolicy(QueuePrecedencePolicy policy) { + kp.put("queuePrecedencePolicy",policy); + } + + /** precedence rank at or below which queues are not crawled */ + protected int precedenceFloor = 255; + public int getPrecedenceFloor() { + return this.precedenceFloor; + } + public void setPrecedenceFloor(int floor) { + this.precedenceFloor = floor; + } + + /** URI precedence assignment policy to use. */ + { + setUriPrecedencePolicy(new CostUriPrecedencePolicy()); + } + public UriPrecedencePolicy getUriPrecedencePolicy() { + return (UriPrecedencePolicy) kp.get("uriPrecedencePolicy"); + } + public void setUriPrecedencePolicy(UriPrecedencePolicy policy) { + kp.put("uriPrecedencePolicy",policy); + } + + /** All known queues. + */ + protected Map allQueues = null; + // of classKey -> ClassKeyQueue + + /** + * All per-class queues whose first item may be handed out. + * Linked-list of keys for the queues. + */ + protected BlockingQueue readyClassQueues; + + /** all per-class queues from whom a URI is outstanding */ + protected Bag inProcessQueues = + BagUtils.synchronizedBag(new HashBag()); // of ClassKeyQueue + + /** + * All per-class queues held in snoozed state, sorted by wake time. + */ + transient protected DelayQueue snoozedClassQueues; + + transient protected WorkQueue longestActiveQueue = null; + + protected int highestPrecedenceWaiting = Integer.MAX_VALUE; + + /** The UriUniqFilter to use, tracking those UURIs which are + * already in-process (or processed), and thus should not be + * rescheduled. Also known as the 'alreadyIncluded' or + * 'alreadySeen' structure */ + protected UriUniqFilter uriUniqFilter; + public UriUniqFilter getUriUniqFilter() { + return this.uriUniqFilter; + } + @Autowired + public void setUriUniqFilter(UriUniqFilter uriUniqFilter) { + this.uriUniqFilter = uriUniqFilter; + } + + /** + * Constructor. + */ + public WorkQueueFrontier() { + super(); + } + + public void start() { + super.start(); + uriUniqFilter.setDestination(this); + + try { + initInternalQueues(false); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + + /** + * Initializes internal queues. May decide to keep all queues in memory based on + * {@link QueueAssignmentPolicy#maximumNumberOfKeys}. Otherwise invokes + * {@link #initAllQueues()} to actually set up the queues. + * + * Subclasses should invoke this method with recycle set to "true" in + * a private readObject method, to restore queues after a checkpoint. + * + * @param recycle + * @throws IOException + * @throws DatabaseException + */ + protected void initInternalQueues(boolean recycle) + throws IOException, DatabaseException { + if (workQueueDataOnDisk() + && getQueueAssignmentPolicy().maximumNumberOfKeys() >= 0 + && getQueueAssignmentPolicy().maximumNumberOfKeys() <= + MAX_QUEUES_TO_HOLD_ALLQUEUES_IN_MEMORY) { + this.allQueues = Collections.synchronizedMap( + new HashMap()); + } else { + this.initAllQueues(); + } + this.initOtherQueues(recycle); + } + + /** + * Initialize the allQueues field in an implementation-appropriate + * way. + * @throws DatabaseException + */ + protected abstract void initAllQueues() throws DatabaseException; + + /** + * Initialize all other internal queues in an implementation-appropriate + * way. + * @throws DatabaseException + */ + protected abstract void initOtherQueues(boolean recycle) throws DatabaseException; + + /* (non-Javadoc) + * @see org.archive.crawler.frontier.AbstractFrontier#crawlEnded(java.lang.String) + */ + public void close() { + // Cleanup. CrawlJobs persist after crawl has finished so undo any + // references. + if (this.uriUniqFilter != null) { + this.uriUniqFilter.close(); +// this.alreadyIncluded = null; + } + +// this.queueAssignmentPolicy = null; + + try { + closeQueue(); + } catch (IOException e) { + // FIXME exception handling + e.printStackTrace(); + } + + this.allQueues.clear(); + } + + /** + * Accept the given CrawlURI for scheduling, as it has + * passed the alreadyIncluded filter. + * + * Choose a per-classKey queue and enqueue it. If this + * item has made an unready queue ready, place that + * queue on the readyClassQueues queue. + * @param caUri CrawlURI. + */ + protected void processScheduleAlways(CrawlURI curi) { + assert Thread.currentThread() == managerThread; + assert KeyedProperties.overridesActiveFrom(curi); + + prepForFrontier(curi); + applySpecialHandling(curi); + sendToQueue(curi); + } + + /** + * Arrange for the given CrawlURI to be visited, if it is not + * already scheduled/completed. + * + * @see org.archive.crawler.framework.Frontier#schedule(org.archive.crawler.datamodel.CrawlURI) + */ + protected void processScheduleIfUnique(CrawlURI curi) { + assert Thread.currentThread() == managerThread; + assert KeyedProperties.overridesActiveFrom(curi); + + // Canonicalization may set forceFetch flag. See + // #canonicalization(CrawlURI) javadoc for circumstance. + String canon = canonicalize(curi); + if (curi.forceFetch()) { + uriUniqFilter.addForce(canon, curi); + } else { + uriUniqFilter.add(canon, curi); + } + } + + /* (non-Javadoc) + * @see org.archive.crawler.frontier.AbstractFrontier#asCrawlUri(org.archive.crawler.datamodel.CrawlURI) + */ + protected void prepForFrontier(CrawlURI caUri) { + super.prepForFrontier(caUri); + // force cost to be calculated, pre-insert + getCost(caUri); + // set + // TODO:SPRINGY set overrides by curi? + getUriPrecedencePolicy().uriScheduled(caUri); + } + + /** + * Send a CrawlURI to the appropriate subqueue. + * + * @param curi + */ + protected void sendToQueue(CrawlURI curi) { + assert Thread.currentThread() == managerThread; + + WorkQueue wq = getQueueFor(curi); + int originalPrecedence = wq.getPrecedence(); + + wq.enqueue(this, curi); + // Update recovery log. + doJournalAdded(curi); + + if(wq.isRetired()) { + return; + } + incrementQueuedUriCount(); + if(wq.isHeld()) { + if(wq.isActive()) { + // queue active -- promote will be handled ok by normal cycling + // do nothing + } else { + // queue is already in a waiting inactive queue; update + int currentPrecedence = wq.getPrecedence(); + if(currentPrecedence < originalPrecedence ) { + // queue bumped up; adjust ordering + deactivateQueue(wq); + // this intentionally places queue in duplicate inactiveQueue\ + // only when it comes off the right queue will it activate; + // otherwise it reenqueues to right inactive queue, if not + // already there (see activateInactiveQueue()) + if(logger.isLoggable(Level.FINE)) { + logger.log(Level.FINE, + "queue re-deactivated to p" +currentPrecedence + + ": " + wq.getClassKey()); + } + } else { + // queue bumped down or stayed same; + // do nothing until it comes up + } + } + } else { + // begin juggling queue between internal ordering structures + wq.setHeld(); + if(getHoldQueues()) { + deactivateQueue(wq); + } else { + replenishSessionBalance(wq); + readyQueue(wq); + } + } + WorkQueue laq = longestActiveQueue; + if(((laq==null) || wq.getCount() > laq.getCount())) { + longestActiveQueue = wq; + } + + } + + /** + * Put the given queue on the readyClassQueues queue + * @param wq + */ + private void readyQueue(WorkQueue wq) { + assert Thread.currentThread() == managerThread; + + try { + wq.setActive(this, true); + readyClassQueues.put(wq.getClassKey()); + if(logger.isLoggable(Level.FINE)) { + logger.log(Level.FINE, + "queue readied: " + wq.getClassKey()); + } + } catch (InterruptedException e) { + e.printStackTrace(); + System.err.println("unable to ready queue "+wq); + // propagate interrupt up + throw new RuntimeException(e); + } + } + + /** + * Put the given queue on the inactiveQueues queue + * @param wq + */ + private void deactivateQueue(WorkQueue wq) { + assert Thread.currentThread() == managerThread; + + wq.setSessionBalance(0); // zero out session balance + int precedence = wq.getPrecedence(); + if(!wq.getOnInactiveQueues().contains(precedence)) { + // not already on target, add + Queue inactiveQueues = + getInactiveQueuesForPrecedence(precedence); + inactiveQueues.add(wq.getClassKey()); + wq.getOnInactiveQueues().add(precedence); + if(wq.getPrecedence() < highestPrecedenceWaiting ) { + highestPrecedenceWaiting = wq.getPrecedence(); + } + if(logger.isLoggable(Level.FINE)) { + logger.log(Level.FINE, + "queue deactivated to p" + precedence + + ": " + wq.getClassKey()); + } + } else { + if(logger.isLoggable(Level.FINE)) { + logger.log(Level.FINE, + "queue already p" + precedence+": " + wq.getClassKey()); + } + } + wq.setActive(this, false); + } + + /** + * Get the queue of inactive uri-queue names at the given precedence. + * + * @param precedence + * @return queue of inacti + */ + protected Queue getInactiveQueuesForPrecedence(int precedence) { + Map> inactiveQueuesByPrecedence = + getInactiveQueuesByPrecedence(); + Queue candidate = inactiveQueuesByPrecedence.get(precedence); + if(candidate==null) { + candidate = createInactiveQueueForPrecedence(precedence); + inactiveQueuesByPrecedence.put(precedence,candidate); + } + return candidate; + } + + /** + * Return a sorted map of all inactive queues, keyed by precedence + * @return SortedMap> of inactiveQueues + */ + abstract SortedMap> getInactiveQueuesByPrecedence(); + + /** + * Create an inactiveQueue to hold queue names at the given precedence + * @param precedence + * @return Queue for names of inactive queues + */ + abstract Queue createInactiveQueueForPrecedence(int precedence); + + /** + * Put the given queue on the retiredQueues queue + * @param wq + */ + private void retireQueue(WorkQueue wq) { + assert Thread.currentThread() == managerThread; + + getRetiredQueues().add(wq.getClassKey()); + decrementQueuedCount(wq.getCount()); + wq.setRetired(true); + if(logger.isLoggable(Level.FINE)) { + logger.log(Level.FINE, + "queue retired: " + wq.getClassKey()); + } + wq.setActive(this, false); + } + + /** + * Return queue of all retired queue names. + * + * @return Queue of retired queue names + */ + abstract Queue getRetiredQueues(); + + /** + * Accomodate any changes in settings. + */ + public void reconsiderRetiredQueues() { + + // The rules for a 'retired' queue may have changed; so, + // unretire all queues to 'inactive'. If they still qualify + // as retired/overbudget next time they come up, they'll + // be re-retired; if not, they'll get a chance to become + // active under the new rules. + + // TODO: Only do this when necessary. + + Object key = getRetiredQueues().poll(); + while (key != null) { + WorkQueue q = (WorkQueue)this.allQueues.get(key); + if(q != null) { + unretireQueue(q); + } + key = getRetiredQueues().poll(); + } + } + /** + * Restore a retired queue to the 'inactive' state. + * + * @param q + */ + private void unretireQueue(WorkQueue q) { + assert Thread.currentThread() == managerThread; + + deactivateQueue(q); + q.setRetired(false); + incrementQueuedUriCount(q.getCount()); + } + + /** + * Return the work queue for the given CrawlURI's classKey. URIs + * are ordered and politeness-delayed within their 'class'. + * If the requested queue is not found, a new instance is created. + * + * @param curi CrawlURI to base queue on + * @return the found or created ClassKeyQueue + */ + protected abstract WorkQueue getQueueFor(CrawlURI curi); + + /** + * Return the work queue for the given classKey, or null + * if no such queue exists. + * + * @param classKey key to look for + * @return the found WorkQueue + */ + protected abstract WorkQueue getQueueFor(String classKey); + + /** + * Return the next CrawlURI eligible to be processed (and presumably + * visited/fetched) by a a worker thread. + * + * Relies on the readyClassQueues having been loaded with + * any work queues that are eligible to provide a URI. + * + * @return next CrawlURI eligible to be processed, or null if none available + * + * @see org.archive.crawler.framework.Frontier#next() + */ + protected CrawlURI findEligibleURI() { + assert Thread.currentThread() == managerThread; + // wake any snoozed queues + wakeQueues(); + // activate enough inactive queues to fill outbound + int activationsWanted = + outbound.remainingCapacity() - readyClassQueues.size(); + while(activationsWanted > 0 + && !getInactiveQueuesByPrecedence().isEmpty() + && highestPrecedenceWaiting < getPrecedenceFloor()) { + activateInactiveQueue(); + activationsWanted--; + } + + // find a non-empty ready queue, if any + // TODO: refactor to untangle these loops, early-exits, etc! + WorkQueue readyQ = null; + findauri: while(true) { + findaqueue: do { + String key = readyClassQueues.poll(); + if(key== null) { + // no ready queues + break; + } + readyQ = getQueueFor(key); + if(readyQ==null) { + // readyQ key wasn't in all queues: unexpected + logger.severe("Key "+ key + + " in readyClassQueues but not allQueues"); + break findaqueue; + } + if(readyQ.getCount()==0) { + // readyQ is empty and ready: it's exhausted + // release held status, allowing any subsequent + // enqueues to again put queue in ready + readyQ.clearHeld(); + readyQ = null; + } + } while (readyQ == null); + + if (readyQ == null) { + break findauri; + } + + assert !inProcessQueues.contains(readyQ) : "double activation"; + returnauri: while(true) { // loop left by explicit return or break on empty + CrawlURI curi = null; + curi = readyQ.peek(this); + if(curi == null) { + // should not reach + logger.severe("No CrawlURI from ready non-empty queue " + + readyQ.classKey + "\n" + + readyQ.singleLineLegend() + "\n" + + readyQ.singleLineReport() + "\n"); + break returnauri; + } + + // from queues, override names persist but not map source + curi.setOverlayMapsSource(sheetOverlaysManager); + // check if curi belongs in different queue + String currentQueueKey; + try { + KeyedProperties.loadOverridesFrom(curi); + currentQueueKey = getClassKey(curi); + } finally { + KeyedProperties.clearOverridesFrom(curi); + } + if (currentQueueKey.equals(curi.getClassKey())) { + // curi was in right queue, emit + noteAboutToEmit(curi, readyQ); + inProcessQueues.add(readyQ); + return curi; + } + // URI's assigned queue has changed since it + // was queued (eg because its IP has become + // known). Requeue to new queue. + readyQ.dequeue(this,curi); + doJournalRelocated(curi); + curi.setClassKey(currentQueueKey); + decrementQueuedCount(1); + curi.setHolderKey(null); + sendToQueue(curi); + if(readyQ.getCount()==0) { + // readyQ is empty and ready: it's exhausted + // release held status, allowing any subsequent + // enqueues to again put queue in ready + readyQ.clearHeld(); + readyQ = null; + continue findauri; + } + } + + } + + if(inProcessQueues.size()==0) { + // Nothing was ready or in progress or imminent to wake; ensure + // any piled-up pending-scheduled URIs are considered + uriUniqFilter.requestFlush(); + } + + // never return null if there are any eligible inactives + if(getTotalEligibleInactiveQueues()>0) { + return findEligibleURI(); + } + + // nothing eligible + return null; + } + + /** + * Return the 'cost' of a CrawlURI (how much of its associated + * queue's budget it depletes upon attempted processing) + * + * @param curi + * @return the associated cost + */ + private int getCost(CrawlURI curi) { + assert KeyedProperties.overridesActiveFrom(curi); + + int cost = curi.getHolderCost(); + if (cost == CrawlURI.UNCALCULATED) { + //TODO:SPRINGY push overrides by curi + cost = getCostAssignmentPolicy().costOf(curi); + curi.setHolderCost(cost); + } + return cost; + } + + /** + * Activate an inactive queue, if any are available. + */ + private void activateInactiveQueue() { + assert Thread.currentThread() == managerThread; + + SortedMap> inactiveQueuesByPrecedence = + getInactiveQueuesByPrecedence(); + + int targetPrecedence = highestPrecedenceWaiting; + Queue inactiveQueues = inactiveQueuesByPrecedence.get( + targetPrecedence); + + Object key = inactiveQueues.poll(); + assert key != null : "empty precedence queue in map"; + + if(inactiveQueues.isEmpty()) { + updateHighestWaiting(targetPrecedence+1); + } + + WorkQueue candidateQ = (WorkQueue) this.allQueues.get(key); + + assert candidateQ != null : "missing uri work queue"; + + boolean was = candidateQ.getOnInactiveQueues().remove(targetPrecedence); + + assert was : "queue didn't know it was in "+targetPrecedence+" inactives"; + + if(candidateQ.isActive()) { + // queue had been multiply-scheduled due to changing precedence + // already active, so ignore this activation + if(logger.isLoggable(Level.FINE)) { + logger.log(Level.FINE, + "queue activated+ignored/active from p" + targetPrecedence + + ": " + candidateQ.getClassKey()); + } + return; + } + + if(candidateQ.getPrecedence() < targetPrecedence) { + // queue moved up; do nothing (already handled) + if(logger.isLoggable(Level.FINE)) { + logger.log(Level.FINE, + "queue activated+ignored/higher from p" + + targetPrecedence + + ": " + candidateQ.getClassKey() + + " ("+candidateQ.getPrecedence() + ") "); + } + return; + } + if(candidateQ.getPrecedence() > targetPrecedence) { + // queue moved down; deactivate to new level + if(logger.isLoggable(Level.FINE)) { + logger.log(Level.FINE, + "queue activated+deactivated from p" + targetPrecedence + + ": " + candidateQ.getClassKey()); + } + deactivateQueue(candidateQ); + return; + } + replenishSessionBalance(candidateQ); + if (candidateQ.isOverBudget()) { + // if still over-budget after an activation & replenishing, + // retire + retireQueue(candidateQ); + return; + } +// long now = System.currentTimeMillis(); +// long delay_ms = candidateQ.getWakeTime() - now; +// if (delay_ms > 0) { +// // queue still due for snoozing +// snoozeQueue(candidateQ, now, delay_ms); +// return; +// } + candidateQ.setWakeTime(0); // clear obsolete wake time, if any + readyQueue(candidateQ); + } + + /** + * Recalculate the value of thehighest-precedence queue waiting + * among inactive queues. + * + * @param startFrom start looking at this precedence value + */ + protected void updateHighestWaiting(int startFrom) { + // probe for new highestWaiting + for(int precedenceKey : getInactiveQueuesByPrecedence().tailMap(startFrom).keySet()) { + if(!getInactiveQueuesByPrecedence().get(precedenceKey).isEmpty()) { + highestPrecedenceWaiting = precedenceKey; + return; + } + } + // nothing waiting + highestPrecedenceWaiting = Integer.MAX_VALUE; + } + + /** + * Replenish the budget of the given queue by the appropriate amount. + * + * @param queue queue to replenish + */ + private void replenishSessionBalance(WorkQueue queue) { + assert queue.peekItem == null : "unexpected peekItem set"; + // get a CrawlURI for override context purposes + CrawlURI contextUri = queue.peek(this); + if(contextUri == null) { + // use globals TODO: fix problems this will cause if + // global total budget < override on empty queue + queue.setSessionBalance(getBalanceReplenishAmount()); + queue.setTotalBudget(getQueueTotalBudget()); + return; + } + // TODO: consider confusing cross-effects of this and IP-based politeness + + contextUri.setOverlayMapsSource(sheetOverlaysManager); + try { + // TODO:SPRINGY set override + KeyedProperties.loadOverridesFrom(contextUri); + + //queue.setSessionBalance(contextUri.get(this, BALANCE_REPLENISH_AMOUNT)); + queue.setSessionBalance(getBalanceReplenishAmount()); + + // reset total budget (it may have changed) + // TODO: is this the best way to be sensitive to potential mid-crawl changes + // TODO:SPRINGY set override + //long totalBudget = contextUri.get(this, QUEUE_TOTAL_BUDGET); + long totalBudget = getQueueTotalBudget(); + queue.setTotalBudget(totalBudget); + queue.unpeek(contextUri); // don't insist on that URI being next released + } finally { + KeyedProperties.clearOverridesFrom(contextUri); + } + } + + /** + * Enqueue the given queue to either readyClassQueues or inactiveQueues, + * as appropriate. + * + * @param wq + */ + private void reenqueueQueue(WorkQueue wq) { + //TODO:SPRINGY set overrides by queue? + getQueuePrecedencePolicy().queueReevaluate(wq); + if (logger.isLoggable(Level.FINE)) { + logger.fine("queue reenqueued: " + + wq.getClassKey()); + } + if(highestPrecedenceWaiting < wq.getPrecedence() + || (wq.isOverBudget() && highestPrecedenceWaiting <= wq.getPrecedence()) + || wq.getPrecedence() >= getPrecedenceFloor()) { + // if still over budget, deactivate + deactivateQueue(wq); + } else { + readyQueue(wq); + } + } + + /* (non-Javadoc) + * @see org.archive.crawler.frontier.AbstractFrontier#getMaxInWait() + */ + @Override + protected long getMaxInWait() { + Delayed next = snoozedClassQueues.peek(); + return next == null ? 60000 : next.getDelay(TimeUnit.MILLISECONDS); + } + + /** + * Wake any queues sitting in the snoozed queue whose time has come. + */ + protected void wakeQueues() { + DelayedWorkQueue waked; + while((waked = snoozedClassQueues.poll())!=null) { + WorkQueue queue = waked.getWorkQueue(); + queue.setWakeTime(0); + reenqueueQueue(queue); + } + } + + /** + * Note that the previously emitted CrawlURI has completed + * its processing (for now). + * + * The CrawlURI may be scheduled to retry, if appropriate, + * and other related URIs may become eligible for release + * via the next next() call, as a result of finished(). + * + * (non-Javadoc) + * @see org.archive.crawler.framework.Frontier#finished(org.archive.crawler.datamodel.CrawlURI) + */ + protected void processFinish(CrawlURI curi) { + assert Thread.currentThread() == managerThread; + + long now = System.currentTimeMillis(); + + curi.incrementFetchAttempts(); + logNonfatalErrors(curi); + WorkQueue wq = (WorkQueue) curi.getHolder(); + assert (wq.peek(this) == curi) : "unexpected peek " + wq; + inProcessQueues.remove(wq, 1); + + if(includesRetireDirective(curi)) { + // CrawlURI is marked to trigger retirement of its queue + curi.processingCleanup(); + wq.unpeek(curi); + wq.update(this, curi); // rewrite any changes + retireQueue(wq); + return; + } + + if (needsRetrying(curi)) { + // Consider errors which can be retried, leaving uri atop queue + if(curi.getFetchStatus()!=S_DEFERRED) { + wq.expend(getCost(curi)); // all retries but DEFERRED cost + } + long delay_sec = retryDelayFor(curi); + curi.processingCleanup(); // lose state that shouldn't burden retry + + wq.unpeek(curi); + // TODO: consider if this should happen automatically inside unpeek() + wq.update(this, curi); // rewrite any changes + if (delay_sec > 0) { + long delay_ms = delay_sec * 1000; + snoozeQueue(wq, now, delay_ms); + } else { + reenqueueQueue(wq); + } + + // Let everyone interested know that it will be retried. + appCtx.publishEvent( + new CrawlURIDispositionEvent(this,curi,DEFERRED_FOR_RETRY)); + doJournalRescheduled(curi); + return; + } + + // Curi will definitely be disposed of without retry, so remove from queue + wq.dequeue(this,curi); + decrementQueuedCount(1); + log(curi); + + if (curi.isSuccess()) { + totalProcessedBytes += curi.getRecordedSize(); + incrementSucceededFetchCount(); + // Let everyone know in case they want to do something before we strip the curi. + appCtx.publishEvent( + new CrawlURIDispositionEvent(this,curi,SUCCEEDED)); + doJournalFinishedSuccess(curi); + wq.expend(getCost(curi)); // successes cost + } else if (isDisregarded(curi)) { + // Check for codes that mean that while we the crawler did + // manage to schedule it, it must be disregarded for some reason. + incrementDisregardedUriCount(); + // Let interested listeners know of disregard disposition. + appCtx.publishEvent( + new CrawlURIDispositionEvent(this,curi,DISREGARDED)); + doJournalDisregarded(curi); + // if exception, also send to crawlErrors + if (curi.getFetchStatus() == S_RUNTIME_EXCEPTION) { + Object[] array = { curi }; + loggerModule.getRuntimeErrors().log(Level.WARNING, curi.getUURI() + .toString(), array); + } + // TODO: consider reinstating forget-uri + } else { + // In that case FAILURE, note & log + //Let interested listeners know of failed disposition. + appCtx.publishEvent( + new CrawlURIDispositionEvent(this,curi,FAILED)); + // if exception, also send to crawlErrors + if (curi.getFetchStatus() == S_RUNTIME_EXCEPTION) { + Object[] array = { curi }; + this.loggerModule.getRuntimeErrors().log(Level.WARNING, curi.getUURI() + .toString(), array); + } + incrementFailedFetchCount(); + // let queue note error + //TODO:SPRINGY set overrides by curi or wq? + assert KeyedProperties.overridesActiveFrom(curi); + + wq.noteError(getErrorPenaltyAmount()); + doJournalFinishedFailure(curi); + wq.expend(getCost(curi)); // failures cost + } + + long delay_ms = politenessDelayFor(curi); + + if (delay_ms > 0) { + snoozeQueue(wq,now,delay_ms); + } else { + reenqueueQueue(wq); + } + + + curi.stripToMinimal(); + curi.processingCleanup(); + + } + + private boolean includesRetireDirective(CrawlURI curi) { + return curi.containsDataKey(A_FORCE_RETIRE) + && (Boolean)curi.getData().get(A_FORCE_RETIRE); + } + + /** + * Place the given queue into 'snoozed' state, ineligible to + * supply any URIs for crawling, for the given amount of time. + * + * @param wq queue to snooze + * @param now time now in ms + * @param delay_ms time to snooze in ms + */ + private void snoozeQueue(WorkQueue wq, long now, long delay_ms) { + long nextTime = now + delay_ms; + wq.setWakeTime(nextTime); +// long snoozeToInactiveDelayMs = get(SNOOZE_DEACTIVATE_MS); +// if (delay_ms > snoozeToInactiveDelayMs && !inactiveQueues.isEmpty()) { +// deactivateQueue(wq); +// } else { + snoozedClassQueues.add(new DelayedWorkQueue(wq)); +// } + } + + /** + * Forget the given CrawlURI. This allows a new instance + * to be created in the future, if it is reencountered under + * different circumstances. + * + * @param curi The CrawlURI to forget + */ + protected void forget(CrawlURI curi) { + logger.finer("Forgetting " + curi); + uriUniqFilter.forget(canonicalize(curi.getUURI()), curi); + } + + /** (non-Javadoc) + * @see org.archive.crawler.framework.Frontier#discoveredUriCount() + */ + public long discoveredUriCount() { + return (this.uriUniqFilter != null)? this.uriUniqFilter.count(): 0; + } + + /** + * @param match String to match. + * @return Number of items deleted. + */ + public long deleteURIs(String queueRegex, String uriRegex) { + long count = 0; + // TODO: DANGER/ values() may not work right from CachedBdbMap + Pattern queuePat = Pattern.compile(queueRegex); + for (String qname: allQueues.keySet()) { + if (queuePat.matcher(qname).matches()) { + WorkQueue wq = getQueueFor(qname); + wq.unpeek(null); + count += wq.deleteMatching(this, uriRegex); + } + } + decrementQueuedCount(count); + return count; + } + + // + // Reporter implementation + // + + public static String STANDARD_REPORT = "standard"; + public static String ALL_NONEMPTY = "nonempty"; + public static String ALL_QUEUES = "all"; + protected static String[] REPORTS = {STANDARD_REPORT,ALL_NONEMPTY,ALL_QUEUES}; + + public String[] getReports() { + return REPORTS; + } + + /** + * @param w Where to write to. + */ + public void singleLineReportTo(PrintWriter w) { + if (this.allQueues == null) { + return; + } + int allCount = allQueues.size(); + int inProcessCount = inProcessQueues.uniqueSet().size(); + int readyCount = readyClassQueues.size(); + int snoozedCount = snoozedClassQueues.size(); + int activeCount = inProcessCount + readyCount + snoozedCount; + int inactiveCount = getTotalEligibleInactiveQueues(); + int ineligibleCount = getTotalIneligibleInactiveQueues(); + int retiredCount = getRetiredQueues().size(); + int exhaustedCount = + allCount - activeCount - inactiveCount - retiredCount; + int inCount = inbound.size(); + int outCount = outbound.size(); + State last = lastReachedState; + w.print(allCount); + w.print(" URI queues: "); + w.print(activeCount); + w.print(" active ("); + w.print(inProcessCount); + w.print(" in-process; "); + w.print(readyCount); + w.print(" ready; "); + w.print(snoozedCount); + w.print(" snoozed); "); + w.print(inactiveCount); + w.print(" inactive; "); + w.print(ineligibleCount); + w.print(" ineligible; "); + w.print(retiredCount); + w.print(" retired; "); + w.print(exhaustedCount); + w.print(" exhausted"); + w.print(" ["+last+ ": "+inCount+" in, "+outCount+" out]"); + w.flush(); + } + + /** + * Total of all URIs in inactive queues at all precedences + * @return int total + */ + protected int getTotalInactiveQueues() { + return tallyInactiveTotals(getInactiveQueuesByPrecedence()); + } + + /** + * Total of all URIs in inactive queues at precedences above the floor + * @return int total + */ + protected int getTotalEligibleInactiveQueues() { + return tallyInactiveTotals( + getInactiveQueuesByPrecedence().headMap(getPrecedenceFloor())); + } + + /** + * Total of all URIs in inactive queues at precedences at or below the floor + * @return int total + */ + protected int getTotalIneligibleInactiveQueues() { + return tallyInactiveTotals( + getInactiveQueuesByPrecedence().tailMap(getPrecedenceFloor())); + } + + /** + * @param iqueue + * @return + */ + private int tallyInactiveTotals(SortedMap> iqueues) { + int inactiveCount = 0; + for(Queue q : iqueues.values()) { + inactiveCount += q.size(); + } + return inactiveCount; + } + + /* (non-Javadoc) + * @see org.archive.util.Reporter#singleLineLegend() + */ + public String singleLineLegend() { + return "total active in-process ready snoozed inactive retired exhausted"; + } + + /** + * This method compiles a human readable report on the status of the frontier + * at the time of the call. + * @param name Name of report. + * @param writer Where to write to. + */ + public synchronized void reportTo(String name, PrintWriter writer) { + if(ALL_NONEMPTY.equals(name)) { + allNonemptyReportTo(writer); + return; + } + if(ALL_QUEUES.equals(name)) { + allQueuesReportTo(writer); + return; + } + if(name!=null && !STANDARD_REPORT.equals(name)) { + writer.print(name); + writer.print(" unavailable; standard report:\n"); + } + standardReportTo(writer); + } + + /** Compact report of all nonempty queues (one queue per line) + * + * @param writer + */ + private void allNonemptyReportTo(PrintWriter writer) { + ArrayList inProcessQueuesCopy; + synchronized(this.inProcessQueues) { + // grab a copy that will be stable against mods for report duration + @SuppressWarnings("unchecked") + Collection inProcess = this.inProcessQueues; + inProcessQueuesCopy = new ArrayList(inProcess); + } + writer.print("\n -----===== IN-PROCESS QUEUES =====-----\n"); + queueSingleLinesTo(writer, inProcessQueuesCopy.iterator()); + + writer.print("\n -----===== READY QUEUES =====-----\n"); + queueSingleLinesTo(writer, this.readyClassQueues.iterator()); + + writer.print("\n -----===== SNOOZED QUEUES =====-----\n"); + queueSingleLinesTo(writer, this.snoozedClassQueues.iterator()); + + writer.print("\n -----===== INACTIVE QUEUES =====-----\n"); + for(Queue inactiveQueues : getInactiveQueuesByPrecedence().values()) { + queueSingleLinesTo(writer, inactiveQueues.iterator()); + } + + writer.print("\n -----===== RETIRED QUEUES =====-----\n"); + queueSingleLinesTo(writer, getRetiredQueues().iterator()); + } + + /** Compact report of all nonempty queues (one queue per line) + * + * @param writer + */ + private void allQueuesReportTo(PrintWriter writer) { + queueSingleLinesTo(writer, allQueues.keySet().iterator()); + } + + /** + * Writer the single-line reports of all queues in the + * iterator to the writer + * + * @param writer to receive report + * @param iterator over queues of interest. + */ + private void queueSingleLinesTo(PrintWriter writer, Iterator iterator) { + Object obj; + WorkQueue q; + boolean legendWritten = false; + while( iterator.hasNext()) { + obj = iterator.next(); + if (obj == null) { + continue; + } + if(obj instanceof WorkQueue) { + q = (WorkQueue)obj; + } else if (obj instanceof DelayedWorkQueue) { + q = ((DelayedWorkQueue)obj).getWorkQueue(); + } else { + try { + q = (WorkQueue)this.allQueues.get(obj); + } catch (ClassCastException cce) { + logger.log(Level.SEVERE,"not convertible to workqueue:"+obj,cce); + q = null; + } + } + + if(q == null) { + writer.print(" ERROR: "+obj); + } + if(!legendWritten) { + writer.println(q.singleLineLegend()); + legendWritten = true; + } + q.singleLineReportTo(writer); + } + } + + /** + * @param w Writer to print to. + */ + private void standardReportTo(PrintWriter w) { + int allCount = allQueues.size(); + int inProcessCount = inProcessQueues.uniqueSet().size(); + int readyCount = readyClassQueues.size(); + int snoozedCount = snoozedClassQueues.size(); + int activeCount = inProcessCount + readyCount + snoozedCount; + int inactiveCount = getTotalInactiveQueues(); + int retiredCount = getRetiredQueues().size(); + int exhaustedCount = + allCount - activeCount - inactiveCount - retiredCount; + + w.print("Frontier report - "); + w.print(ArchiveUtils.get12DigitDate()); + w.print("\n"); + w.print(" Job being crawled: "); + w.print(controller.getMetadata().getJobName()); + w.print("\n"); + w.print("\n -----===== STATS =====-----\n"); + w.print(" Discovered: "); + w.print(Long.toString(discoveredUriCount())); + w.print("\n"); + w.print(" Queued: "); + w.print(Long.toString(queuedUriCount())); + w.print("\n"); + w.print(" Finished: "); + w.print(Long.toString(finishedUriCount())); + w.print("\n"); + w.print(" Successfully: "); + w.print(Long.toString(succeededFetchCount())); + w.print("\n"); + w.print(" Failed: "); + w.print(Long.toString(failedFetchCount())); + w.print("\n"); + w.print(" Disregarded: "); + w.print(Long.toString(disregardedUriCount())); + w.print("\n"); + w.print("\n -----===== QUEUES =====-----\n"); + w.print(" Already included size: "); + w.print(Long.toString(uriUniqFilter.count())); + w.print("\n"); + w.print(" pending: "); + w.print(Long.toString(uriUniqFilter.pending())); + w.print("\n"); + w.print("\n All class queues map size: "); + w.print(Long.toString(allCount)); + w.print("\n"); + w.print( " Active queues: "); + w.print(activeCount); + w.print("\n"); + w.print(" In-process: "); + w.print(inProcessCount); + w.print("\n"); + w.print(" Ready: "); + w.print(readyCount); + w.print("\n"); + w.print(" Snoozed: "); + w.print(snoozedCount); + w.print("\n"); + w.print(" Inactive queues: "); + w.print(inactiveCount); + w.print(" ("); + Map> inactives = getInactiveQueuesByPrecedence(); + boolean betwixt = false; + for(Integer k : inactives.keySet()) { + if(betwixt) { + w.print("; "); + } + w.print("p"); + w.print(k); + w.print(": "); + w.print(inactives.get(k).size()); + betwixt = true; + } + w.print(")\n"); + w.print(" Retired queues: "); + w.print(retiredCount); + w.print("\n"); + w.print(" Exhausted queues: "); + w.print(exhaustedCount); + w.print("\n"); + + w.print("\n -----===== MANAGER THREAD =====-----\n"); + ToeThread.reportThread(managerThread, w); + + w.print("\n -----===== IN-PROCESS QUEUES =====-----\n"); + @SuppressWarnings("unchecked") + Collection inProcess = inProcessQueues; + ArrayList copy = extractSome(inProcess, REPORT_MAX_QUEUES); + appendQueueReports(w, copy.iterator(), copy.size(), REPORT_MAX_QUEUES); + + w.print("\n -----===== READY QUEUES =====-----\n"); + appendQueueReports(w, this.readyClassQueues.iterator(), + this.readyClassQueues.size(), REPORT_MAX_QUEUES); + + w.print("\n -----===== SNOOZED QUEUES =====-----\n"); + Transformer ter = + new Transformer() { + public WorkQueue transform(DelayedWorkQueue dwq) { + return dwq.getWorkQueue(); + } + }; + Transform t = + new Transform(snoozedClassQueues, ter); + copy = extractSome(t, REPORT_MAX_QUEUES); + appendQueueReports(w, copy.iterator(), copy.size(), REPORT_MAX_QUEUES); + + WorkQueue longest = longestActiveQueue; + if (longest != null) { + w.print("\n -----===== LONGEST QUEUE =====-----\n"); + longest.reportTo(w); + } + + w.print("\n -----===== INACTIVE QUEUES =====-----\n"); + for(Queue inactiveQueues : getInactiveQueuesByPrecedence().values()) { + appendQueueReports(w, inactiveQueues.iterator(), + inactiveQueues.size(), REPORT_MAX_QUEUES); + } + + w.print("\n -----===== RETIRED QUEUES =====-----\n"); + appendQueueReports(w, getRetiredQueues().iterator(), + getRetiredQueues().size(), REPORT_MAX_QUEUES); + + w.flush(); + } + + + /** + * Extract some of the elements in the given collection to an + * ArrayList. This method synchronizes on the given collection's + * monitor. The returned list will never contain more than the + * specified maximum number of elements. + * + * @param c the collection whose elements to extract + * @param max the maximum number of elements to extract + * @return the extraction + */ + private static ArrayList extractSome(Collection c, int max) { + // Try to guess a sane initial capacity for ArrayList + // Hopefully given collection won't grow more than 10 items + // between now and the synchronized block... + int initial = Math.min(c.size() + 10, max); + int count = 0; + ArrayList list = new ArrayList(initial); + synchronized (c) { + Iterator iter = c.iterator(); + while (iter.hasNext() && (count < max)) { + list.add(iter.next()); + count++; + } + } + return list; + } + + /** + * Append queue report to general Frontier report. + * @param w StringBuffer to append to. + * @param iterator An iterator over + * @param total + * @param max + */ + protected void appendQueueReports(PrintWriter w, Iterator iterator, + int total, int max) { + Object obj; + WorkQueue q; + for(int count = 0; iterator.hasNext() && (count < max); count++) { + obj = iterator.next(); + if (obj == null) { + continue; + } + q = (obj instanceof WorkQueue)? + (WorkQueue)obj: + (WorkQueue)this.allQueues.get(obj); + if(q == null) { + w.print("WARNING: No report for queue "+obj); + } + q.reportTo(w); + } + if(total > max) { + w.print("...and " + (total - max) + " more.\n"); + } + } + + /** + * Force logging, etc. of operator- deleted CrawlURIs + * + * @see org.archive.crawler.framework.Frontier#deleted(org.archive.crawler.datamodel.CrawlURI) + */ + public void deleted(CrawlURI curi) { + //treat as disregarded + appCtx.publishEvent( + new CrawlURIDispositionEvent(this,curi,DISREGARDED)); + log(curi); + incrementDisregardedUriCount(); + curi.stripToMinimal(); + curi.processingCleanup(); + } + + public void considerIncluded(UURI u) { + this.uriUniqFilter.note(canonicalize(u)); + CrawlURI temp = new CrawlURI(u); + applyOverridesTo(temp); + try { + KeyedProperties.loadOverridesFrom(temp); + temp.setClassKey(getClassKey(temp)); + getQueueFor(temp).expend(getCost(temp)); + } finally { + KeyedProperties.clearOverridesFrom(temp); + } + + } + + protected abstract void closeQueue() throws IOException; + + /** + * Returns true if the WorkQueue implementation of this + * Frontier stores its workload on disk instead of relying + * on serialization mechanisms. + * + * TODO: rename! (this is a very misleading name) or kill (don't + * see any implementations that return false) + * + * @return a constant boolean value for this class/instance + */ + protected abstract boolean workQueueDataOnDisk(); + + + public FrontierGroup getGroup(CrawlURI curi) { + return getQueueFor(curi); + } + + + public long averageDepth() { + if(inProcessQueues==null || readyClassQueues==null || snoozedClassQueues==null) { + return 0; + } + int inProcessCount = inProcessQueues.uniqueSet().size(); + int readyCount = readyClassQueues.size(); + int snoozedCount = snoozedClassQueues.size(); + int activeCount = inProcessCount + readyCount + snoozedCount; + int inactiveCount = getTotalInactiveQueues(); + int totalQueueCount = (activeCount+inactiveCount); + return (totalQueueCount == 0) ? 0 : queuedUriCount.get() / totalQueueCount; + } + public float congestionRatio() { + if(inProcessQueues==null || readyClassQueues==null || snoozedClassQueues==null) { + return 0; + } + int inProcessCount = inProcessQueues.uniqueSet().size(); + int readyCount = readyClassQueues.size(); + int snoozedCount = snoozedClassQueues.size(); + int activeCount = inProcessCount + readyCount + snoozedCount; + int eligibleInactiveCount = getTotalEligibleInactiveQueues(); + return (float)(activeCount + eligibleInactiveCount) / (inProcessCount + snoozedCount); + } + public long deepestUri() { + return longestActiveQueue==null ? -1 : longestActiveQueue.getCount(); + } + + /** + * Return whether frontier is exhausted: all crawlable URIs done (none + * waiting or pending). Only gives precise answer inside managerThread. + * + * @see org.archive.crawler.framework.Frontier#isEmpty() + */ + public boolean isEmpty() { + return queuedUriCount.get() == 0 + && uriUniqFilter.pending() == 0 + && inbound.isEmpty(); + } + + /* (non-Javadoc) + * @see org.archive.crawler.frontier.AbstractFrontier#getInProcessCount() + */ + @Override + protected int getInProcessCount() { + return inProcessQueues.size(); + } + + /** + * Custom deserialization: bring in snoozed queues as array of + * their names (aka 'classKeys'). + * @param stream + * @throws IOException + * @throws ClassNotFoundException + */ + private void readObject(java.io.ObjectInputStream stream) + throws IOException, ClassNotFoundException { + stream.defaultReadObject(); + DelayedWorkQueue[] snoozed = (DelayedWorkQueue[])stream.readObject(); + snoozedClassQueues = new DelayQueue( + Arrays.asList(snoozed)); + } + + /** + * Custom serialization: write snoozed queues as array of their + * names (aka 'classKeys'). + * + * @param stream + * @throws IOException + */ + private void writeObject(java.io.ObjectOutputStream stream) + throws IOException { + stream.defaultWriteObject(); + DelayedWorkQueue[] snoozed = snoozedClassQueues.toArray( + new DelayedWorkQueue[0]); + stream.writeObject(snoozed); + } + + + class DelayedWorkQueue implements Delayed, Serializable { + + private static final long serialVersionUID = 1L; + + public String classKey; + public long wakeTime; + + /** + * Something can become a WorkQueue instance. This can be three things: + * + *

    + *
  1. null, if this DelayedWorkQueue instance was recently + * deserialized; + *
  2. A SoftReference<WorkQueue>, if the WorkQueue's waitTime + * exceeded SNOOZE_LONG_MS + *
  3. A hard WorkQueue reference, if the WorkQueue's waitTime did not + * exceed SNOOZE_LONG_MS. Idea here is that we thought we + * needed the WorkQueue soon and didn't want to risk losing the + * instance. + *
+ * + * The {@link #getWorkQueue()} method figures out what to return in + * all three of the above cases. + */ + private transient Object workQueue; + + public DelayedWorkQueue(WorkQueue queue) { + this.classKey = queue.getClassKey(); + this.wakeTime = queue.getWakeTime(); + + this.workQueue = queue; + } + + private void setWorkQueue(WorkQueue queue) { + long wakeTime = queue.getWakeTime(); + long delay = wakeTime - System.currentTimeMillis(); + if (delay > getSnoozeLongMs()) { + this.workQueue = new SoftReference(queue); + } else { + this.workQueue = queue; + } + } + + + public WorkQueue getWorkQueue() { + if (workQueue == null) { + // This is a recently deserialized DelayedWorkQueue instance + WorkQueue result = getQueueFor(classKey); + setWorkQueue(result); + return result; + } + if (workQueue instanceof SoftReference) { + @SuppressWarnings("unchecked") + SoftReference ref = (SoftReference)workQueue; + WorkQueue result = ref.get(); + if (result == null) { + result = getQueueFor(classKey); + } + setWorkQueue(result); + return result; + } + return (WorkQueue)workQueue; + } + + public long getDelay(TimeUnit unit) { + return unit.convert( + wakeTime - System.currentTimeMillis(), + TimeUnit.MILLISECONDS); + } + + public String getClassKey() { + return classKey; + } + + public long getWakeTime() { + return wakeTime; + } + + public void setWakeTime(long time) { + this.wakeTime = time; + } + + public int compareTo(Delayed obj) { + if (this == obj) { + return 0; // for exact identity only + } + DelayedWorkQueue other = (DelayedWorkQueue) obj; + if (wakeTime > other.getWakeTime()) { + return 1; + } + if (wakeTime < other.getWakeTime()) { + return -1; + } + // at this point, the ordering is arbitrary, but still + // must be consistent/stable over time + return this.classKey.compareTo(other.getClassKey()); + } + + } + +} + diff --git a/engine/src/main/java/org/archive/crawler/frontier/ZeroCostAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/ZeroCostAssignmentPolicy.java new file mode 100644 index 00000000..9d309f7d --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/ZeroCostAssignmentPolicy.java @@ -0,0 +1,27 @@ +/* + * Created on Dec 15, 2004 + * + * To change the template for this generated file go to + * Window>Preferences>Java>Code Generation>Code and Comments + */ +package org.archive.crawler.frontier; + +import org.archive.crawler.datamodel.CrawlURI; + +/** + * CostAssignmentPolicy considering all URIs costless -- essentially + * disabling budgetting features. + * + * @author gojomo + */ +public class ZeroCostAssignmentPolicy extends CostAssignmentPolicy { + private static final long serialVersionUID = 1L; + + /* (non-Javadoc) + * @see org.archive.crawler.frontier.CostAssignmentPolicy#costOf(org.archive.crawler.datamodel.CrawlURI) + */ + public int costOf(CrawlURI curi) { + return 0; + } + +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/BaseQueuePrecedencePolicy.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/BaseQueuePrecedencePolicy.java new file mode 100644 index 00000000..b3f124c7 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/BaseQueuePrecedencePolicy.java @@ -0,0 +1,106 @@ +/* BaseQueuePrecedencePolicy +* +* $Id: CostAssignmentPolicy.java 4981 2007-03-12 07:06:01Z paul_jack $ +* +* Created on Nov 17, 2007 +* +* Copyright (C) 2007 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.crawler.frontier.precedence; + +import org.archive.crawler.frontier.WorkQueue; +import org.archive.spring.HasKeyedProperties; +import org.archive.spring.KeyedProperties; + +/** + * QueuePrecedencePolicy that sets a uri-queue's precedence to a configured + * single value. This value may vary for some queues (as in override + * settings sheets) or be changed by an operator mid-crawl (subject to limits + * on when such changes are noted in a uri-queues lifecycle). + * + */ +public class BaseQueuePrecedencePolicy extends QueuePrecedencePolicy +implements HasKeyedProperties { + private static final long serialVersionUID = 8312032856661175869L; + + protected KeyedProperties kp = new KeyedProperties(); + public KeyedProperties getKeyedProperties() { + return kp; + } + + /** constant precedence to assign; default is 1 */ + { + setBasePrecedence(1); + } + public int getBasePrecedence() { + return (Integer) kp.get("basePrecedence"); + } + public void setBasePrecedence(int precedence) { + kp.put("basePrecedence", precedence); + } + + /* (non-Javadoc) + * @see org.archive.crawler.frontier.QueuePrecedencePolicy#queueCreated(org.archive.crawler.frontier.WorkQueue) + */ + @Override + public void queueCreated(WorkQueue wq) { + installProvider(wq); + } + + /** + * Install the appropriate provider helper object into the WorkQueue, + * if necessary. + * + * @param wq target WorkQueue this policy will operate on + */ + protected void installProvider(WorkQueue wq) { + SimplePrecedenceProvider precedenceProvider = + new SimplePrecedenceProvider(calculatePrecedence(wq)); + wq.setPrecedenceProvider(precedenceProvider); + } + + /** + * Calculate the precedence value for the given queue. + * + * @param wq WorkQueue + * @return int precedence value + */ + protected int calculatePrecedence(WorkQueue wq) { + return getBasePrecedence(); + } + + /* (non-Javadoc) + * @see org.archive.crawler.frontier.QueuePrecedencePolicy#queueReevaluate(org.archive.crawler.frontier.WorkQueue) + */ + @Override + public void queueReevaluate(WorkQueue wq) { + PrecedenceProvider precedenceProvider = + wq.getPrecedenceProvider(); + // TODO: consider if this fails to replace provider that is + // a subclass of Simple when necessary + if(precedenceProvider instanceof SimplePrecedenceProvider) { + // reset inside provider + ((SimplePrecedenceProvider)precedenceProvider).setPrecedence( + calculatePrecedence(wq)); + } else { + // replace provider + installProvider(wq); + } + } +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/BaseUriPrecedencePolicy.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/BaseUriPrecedencePolicy.java new file mode 100644 index 00000000..2dd983d4 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/BaseUriPrecedencePolicy.java @@ -0,0 +1,66 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.frontier.precedence; + +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.spring.HasKeyedProperties; +import org.archive.spring.KeyedProperties; + +/** + * UriPrecedencePolicy which assigns URIs a set value (perhaps a overridden + * for different URIs). + */ +public class BaseUriPrecedencePolicy extends UriPrecedencePolicy +implements HasKeyedProperties { + private static final long serialVersionUID = -8247330811715982746L; + + protected KeyedProperties kp = new KeyedProperties(); + public KeyedProperties getKeyedProperties() { + return kp; + } + + /** constant precedence to assign; default is 1 */ + { + setBasePrecedence(1); + } + public int getBasePrecedence() { + return (Integer) kp.get("basePrecedence"); + } + public void setBasePrecedence(int precedence) { + kp.put("basePrecedence", precedence); + } + + /* (non-Javadoc) + * @see org.archive.crawler.frontier.precedence.UriPrecedencePolicy#uriScheduled(org.archive.crawler.datamodel.CrawlURI) + */ + @Override + public void uriScheduled(CrawlURI curi) { + curi.setPrecedence(calculatePrecedence(curi)); + } + + /** + * Calculate the precedence value for the given URI. + * @param curi CrawlURI to evaluate + * @return int precedence for URI + */ + protected int calculatePrecedence(CrawlURI curi) { + return getBasePrecedence(); + } +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/CostUriPrecedencePolicy.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/CostUriPrecedencePolicy.java new file mode 100644 index 00000000..02a64888 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/CostUriPrecedencePolicy.java @@ -0,0 +1,44 @@ +/* CostUriPrecedencePolicy +* +* $Id: CostAssignmentPolicy.java 4981 2007-03-12 07:06:01Z paul_jack $ +* +* Created on Nov 20, 2007 +* +* Copyright (C) 2007 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.crawler.frontier.precedence; + +import org.archive.crawler.datamodel.CrawlURI; + +/** + * UriPrecedencePolicy which sets a URI's precedence to its 'cost' -- which + * simulates the in-queue sorting order in Heritrix 1.x, where cost + * contributed the same bits to the queue-insert-key that precedence now does. + */ +public class CostUriPrecedencePolicy extends UriPrecedencePolicy { + private static final long serialVersionUID = -8164425278358540710L; + + /* (non-Javadoc) + * @see org.archive.crawler.frontier.precedence.UriPrecedencePolicy#uriScheduled(org.archive.crawler.datamodel.CrawlURI) + */ + @Override + public void uriScheduled(CrawlURI curi) { + curi.setPrecedence(curi.getHolderCost()); + } +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/HighestUriQueuePrecedencePolicy.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/HighestUriQueuePrecedencePolicy.java new file mode 100644 index 00000000..d0b5a434 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/HighestUriQueuePrecedencePolicy.java @@ -0,0 +1,130 @@ +/* HighestUriQueuePrecedencePolicy +* +* $Id: CostAssignmentPolicy.java 4981 2007-03-12 07:06:01Z paul_jack $ +* +* Created on Nov 17, 2007 +* +* Copyright (C) 2007 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.crawler.frontier.precedence; + +import java.io.PrintWriter; + +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.crawler.frontier.WorkQueue; +import org.archive.modules.ProcessorURI; +import org.archive.modules.fetcher.FetchStats.Stage; +import org.archive.util.Histotable; + +/** + * QueuePrecedencePolicy that sets a uri-queue's precedence to that of the + * highest URI currently enqueued within itself, added to the configured + * base-precedence. + * + * It does this by maintaining a count of the included URIs at each + * URI-precedence, updated on each URI add or remove. + */ +public class HighestUriQueuePrecedencePolicy extends BaseQueuePrecedencePolicy { + private static final long serialVersionUID = -8652293180921419601L; + + /* (non-Javadoc) + * @see org.archive.crawler.frontier.precedence.BaseQueuePrecedencePolicy#installProvider(org.archive.crawler.frontier.WorkQueue) + */ + @Override + protected void installProvider(WorkQueue wq) { + // TODO:SPRINGY ensure proper override context installed for getBasePrecedence() below + HighestUriPrecedenceProvider provider = new HighestUriPrecedenceProvider(getBasePrecedence()); + wq.setPrecedenceProvider(provider); + } + + /** + * Helper provider for maintaining the tracked distribution of included + * URIs and calculating the queue precedence. + */ + public class HighestUriPrecedenceProvider extends SimplePrecedenceProvider { + private static final long serialVersionUID = 5545297542888582745L; + + protected Histotable enqueuedCounts = new Histotable(); + public HighestUriPrecedenceProvider(int base) { + super(base); + } + + /* (non-Javadoc) + * @see org.archive.crawler.frontier.precedence.PrecedenceProvider#tally(org.archive.modules.ProcessorURI, org.archive.modules.fetcher.FetchStats.Stage) + */ + @Override + public void tally(ProcessorURI puri, Stage stage) { + CrawlURI curi = (CrawlURI)puri; + switch(stage) { + case SCHEDULED: + // enqueued + enqueuedCounts.tally(curi.getPrecedence()); + break; + case SUCCEEDED: + case DISREGARDED: + case FAILED: + // dequeued + enqueuedCounts.tally(curi.getPrecedence(),-1); + break; + case RETRIED: + // do nothing, already tallied + break; + } + } + + /* (non-Javadoc) + * @see org.archive.crawler.frontier.precedence.SimplePrecedenceProvider#getPrecedence() + */ + @Override + public int getPrecedence() { + // base plus highest URI still in queue + Integer delta = (enqueuedCounts.size() > 0) ? enqueuedCounts.firstKey() : 0; + return super.getPrecedence() + delta; + } + + /* (non-Javadoc) + * @see org.archive.crawler.frontier.precedence.PrecedenceProvider#singleLineLegend() + */ + @Override + public String singleLineLegend() { + StringBuilder sb = new StringBuilder(); + sb.append(super.singleLineLegend()); + sb.append(":"); + for(Integer p : enqueuedCounts.keySet()) { + sb.append(" p"); + sb.append(p); + } + return sb.toString(); + } + + /* (non-Javadoc) + * @see org.archive.crawler.frontier.precedence.PrecedenceProvider#singleLineReportTo(java.io.PrintWriter) + */ + @Override + public void singleLineReportTo(PrintWriter writer) { + boolean betwixt = false; + for(Long count : enqueuedCounts.values()) { + if(betwixt) writer.print(" "); + writer.print(count); + betwixt = true; + } + } + + } +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/HopsUriPrecedencePolicy.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/HopsUriPrecedencePolicy.java new file mode 100644 index 00000000..5b349b32 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/HopsUriPrecedencePolicy.java @@ -0,0 +1,58 @@ +/* HopsUriPrecedencePolicy +* +* $Id: CostAssignmentPolicy.java 4981 2007-03-12 07:06:01Z paul_jack $ +* +* Created on Nov 21, 2007 +* +* Copyright (C) 2007 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.crawler.frontier.precedence; + +import org.archive.crawler.datamodel.CrawlURI; + +/** + * UriPrecedencePolicy which assigns URIs a precedence equal to the number + * of hops in its hops-path-from-seed (either all hops or just navlink ('L') + * hops. + */ +public class HopsUriPrecedencePolicy extends BaseUriPrecedencePolicy { + private static final long serialVersionUID = 2602303562177294731L; + + /** whether to count only navlinks ('L'), or all hops */ + { + setNavlinksOnly(true); + } + public boolean getNavlinksOnly() { + return (Boolean) kp.get("navlinksOnly"); + } + public void setNavlinksOnly(boolean navsOnly) { + kp.put("navlinksOnly",navsOnly); + } + + /* (non-Javadoc) + * @see org.archive.crawler.frontier.precedence.BaseUriPrecedencePolicy#calculatePrecedence(org.archive.crawler.datamodel.CrawlURI) + */ + @Override + protected int calculatePrecedence(CrawlURI curi) { + return super.calculatePrecedence(curi) + + ((getNavlinksOnly()) + ? curi.getLinkHopCount() + : curi.getPathFromSeed().length()); + } +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceLoader.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceLoader.java new file mode 100644 index 00000000..82807369 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceLoader.java @@ -0,0 +1,163 @@ +/* PrecedenceLoader.java + * + * Created on Nov 27, 2005 + * + * Copyright (C) 2007 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.crawler.frontier.precedence; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +import static org.archive.crawler.datamodel.CoreAttributeConstants.*; +import org.archive.io.CrawlerJournal; +import org.archive.modules.recrawl.PersistProcessor; +import org.archive.util.bdbje.EnhancedEnvironment; +import org.archive.util.iterator.LineReadingIterator; + +import com.sleepycat.bind.serial.SerialBinding; +import com.sleepycat.bind.serial.StoredClassCatalog; +import com.sleepycat.bind.tuple.StringBinding; +import com.sleepycat.collections.StoredSortedMap; +import com.sleepycat.je.Database; +import com.sleepycat.je.DatabaseException; + + + +/** + * Utility class for loading externally-created URI-precedence values + * into the URI-history database. + * + * TODO: refactor code relied on in PersistProcessor for easier reuse here + * (and elsewhere) + * + * @author gojomo + */ +public class PrecedenceLoader { + + public PrecedenceLoader() { + } + + /** + * Utility main for importing a text file (first argument) with lines of + * the form: + * + * URI [whitespace] precedence + * + * into a BDB-JE environment (second argument, created if necessary). + * + * @param args command-line arguments + * @throws DatabaseException + * @throws IOException + */ + public static void main(String[] args) throws DatabaseException, IOException { + if(args.length==2) { + main2args(args); + } else { + System.out.println("Arguments: "); + System.out.println(" source target"); + System.out.println( + "...where source is a file of lines 'URI precedence' "); + System.out.println( + "and target is a BDB env dir (created if necessary). "); + return; + } + + } + + /** + * Merge the precalculated precedence information in the first argument + * file to the environment in the second environment (path; environment + * will be created if it does not already exist). + * + * @param args command-line arguments + * @throws DatabaseException + * @throws FileNotFoundException + * @throws UnsupportedEncodingException + * @throws IOException + */ + @SuppressWarnings("unchecked") + private static void main2args(String[] args) throws DatabaseException, + FileNotFoundException, UnsupportedEncodingException, IOException { + File source = new File(args[0]); + File env = new File(args[1]); + if(!env.exists()) { + env.mkdirs(); + } + + // setup target environment + EnhancedEnvironment targetEnv = PersistProcessor.setupEnvironment(env); + StoredClassCatalog classCatalog = targetEnv.getClassCatalog(); + Database historyDB = targetEnv.openDatabase( + null, + PersistProcessor.URI_HISTORY_DBNAME, + PersistProcessor.historyDatabaseConfig().toDatabaseConfig()); + StoredSortedMap historyMap = new StoredSortedMap(historyDB, + new StringBinding(), new SerialBinding(classCatalog, + Map.class), true); + + int count = 0; + + if(source.isFile()) { + // scan log, writing to database + BufferedReader br = CrawlerJournal.getBufferedReader(source); + Iterator iter = new LineReadingIterator(br); + while(iter.hasNext()) { + String line = (String) iter.next(); + String[] splits = line.split("\\s"); + String uri = splits[0]; + if(!uri.matches("\\w+:.*")) { + // prepend "http://" + uri = "http://"+uri; + } + String key = PersistProcessor.persistKeyFor(uri); + int precedence = Integer.parseInt(splits[1]); + Map map = (Map)historyMap.get(key); + if (map==null) { + map = new HashMap(); + } + map.put(A_PRECALC_PRECEDENCE, precedence); + historyMap.put(key,map); + count++; + if(count % 100000 == 0) { + System.out.print(count+"... "); + } + } + br.close(); + System.out.println(); + System.out.println(count+" entries loaded"); + } else { + // error + System.err.println("unacceptable source file"); + return; + } + + // cleanup + historyDB.sync(); + historyDB.close(); + targetEnv.close(); + System.out.println(count+" records imported from "+source+" to BDB env "+env); + } +} \ No newline at end of file diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceProvider.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceProvider.java new file mode 100644 index 00000000..d8e4f0c9 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceProvider.java @@ -0,0 +1,84 @@ +/* PrecedenceProvider +* +* $Id: CostAssignmentPolicy.java 4981 2007-03-12 07:06:01Z paul_jack $ +* +* Created on Nov 20, 2007 +* +* Copyright (C) 2007 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.crawler.frontier.precedence; + +import java.io.PrintWriter; +import java.io.Serializable; + +import org.archive.modules.ProcessorURI; +import org.archive.modules.fetcher.FetchStats; +import org.archive.modules.fetcher.FetchStats.Stage; +import org.archive.util.ArchiveUtils; +import org.archive.util.MultiReporter; + +/** + * Parent class for precedence-providers, stateful helpers that can be + * installed in a WorkQueue to implement various queue-precedence policies. + */ +abstract public class PrecedenceProvider implements MultiReporter, +FetchStats.CollectsFetchStats, Serializable { + + abstract public int getPrecedence(); + + /* (non-Javadoc) + * @see org.archive.modules.fetcher.FetchStats.CollectsFetchStats#tally(org.archive.modules.ProcessorURI, org.archive.modules.fetcher.FetchStats.Stage) + */ + public void tally(ProcessorURI curi, Stage stage) { + // by default do nothing; subclasses do more + } + + public String[] getReports() { + // TODO Auto-generated method stub + return null; + } + + /* (non-Javadoc) + * @see org.archive.util.Reporter#reportTo(java.lang.String, java.io.PrintWriter) + */ + public void reportTo(String name, PrintWriter writer) { + // name ignored, only one report + writer.println(singleLineLegend()); + singleLineReportTo(writer); + } + + /* (non-Javadoc) + * @see org.archive.util.Reporter#reportTo(java.io.PrintWriter) + */ + public void reportTo(PrintWriter writer) { + reportTo(null,writer); + } + + public String singleLineLegend() { + return getClass().getSimpleName(); + } + + public String singleLineReport() { + return ArchiveUtils.singleLineReport(this); + } + + public void singleLineReportTo(PrintWriter writer) { + writer.print(getPrecedence()); + } +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/PreloadedUriPrecedencePolicy.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/PreloadedUriPrecedencePolicy.java new file mode 100644 index 00000000..77b1c489 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/PreloadedUriPrecedencePolicy.java @@ -0,0 +1,158 @@ +/* PreloadedUriPrecedencePolicy +* +* $Id: CostAssignmentPolicy.java 4981 2007-03-12 07:06:01Z paul_jack $ +* +* Created on Nov 27, 2007 +* +* Copyright (C) 2007 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.crawler.frontier.precedence; + +import static org.archive.crawler.datamodel.CoreAttributeConstants.A_PRECALC_PRECEDENCE; + +import java.util.Map; + +import org.archive.bdb.BdbModule; +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.recrawl.PersistProcessor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.Lifecycle; + +import com.sleepycat.bind.serial.SerialBinding; +import com.sleepycat.bind.serial.StoredClassCatalog; +import com.sleepycat.bind.tuple.StringBinding; +import com.sleepycat.collections.StoredSortedMap; +import com.sleepycat.je.Database; +import com.sleepycat.je.DatabaseException; + +/** + * UriPrecedencePolicy which assigns URIs a precedence from a value that + * was preloaded for them into the uri-history database. + * + * NOTE: Because this is a Lifecycle bean requiring start and stop, it + * should not be instantiated as an anonymous inner bean. Rather, it + * should be a top-level named bean, then either autowired or placed-by- + * reference into the frontier. + */ +public class PreloadedUriPrecedencePolicy extends BaseUriPrecedencePolicy +implements Lifecycle { + private static final long serialVersionUID = -1474685153995064123L; + + /** Backup URI precedence assignment policy to use. */ + { + setDefaultUriPrecedencePolicy(new BaseUriPrecedencePolicy()); + } + public UriPrecedencePolicy getDefaultUriPrecedencePolicy() { + return (UriPrecedencePolicy) kp.get("defaultUriPrecedencePolicy"); + } + public void setDefaultUriPrecedencePolicy(UriPrecedencePolicy policy) { + kp.put("defaultUriPrecedencePolicy",policy); + } + + // TODO: refactor to better share code with PersistOnlineProcessor + protected BdbModule bdb; + @Autowired + public void setBdbModule(BdbModule bdb) { + this.bdb = bdb; + } + + @SuppressWarnings("unchecked") + protected StoredSortedMap store; + protected Database historyDb; + + @SuppressWarnings("unchecked") + public void start() { + if(isRunning()) { + return; + } + String dbName = PersistProcessor.URI_HISTORY_DBNAME; + StoredSortedMap historyMap; + try { + StoredClassCatalog classCatalog = bdb.getClassCatalog(); + BdbModule.BdbConfig dbConfig = PersistProcessor.historyDatabaseConfig(); + + historyDb = bdb.openDatabase(dbName, dbConfig, true); + historyMap = new StoredSortedMap(historyDb, + new StringBinding(), new SerialBinding(classCatalog, + Map.class), true); + } catch (DatabaseException e) { + throw new RuntimeException(e); + } + store = historyMap; + } + + public boolean isRunning() { + return historyDb != null; + } + + public void stop() { + if(!isRunning()) { + return; + } + // BdbModule will handle closing of DB + historyDb = null; + } + + /* (non-Javadoc) + * @see org.archive.crawler.frontier.precedence.BaseUriPrecedencePolicy#uriScheduled(org.archive.crawler.datamodel.CrawlURI) + */ + @Override + public void uriScheduled(CrawlURI curi) { + int precedence = calculatePrecedence(curi); + if(precedence==0) { + // fall back to configured default policy + getDefaultUriPrecedencePolicy().uriScheduled(curi); + return; + } + curi.setPrecedence(precedence); + + } + + /* (non-Javadoc) + * @see org.archive.crawler.frontier.precedence.BaseUriPrecedencePolicy#calculatePrecedence(org.archive.crawler.datamodel.CrawlURI) + */ + @Override + protected int calculatePrecedence(CrawlURI curi) { + mergePrior(curi); + Integer preloadPrecedence = (Integer) curi.getData().get(A_PRECALC_PRECEDENCE); + if(preloadPrecedence==null) { + return 0; + } + return super.calculatePrecedence(curi) + preloadPrecedence; + } + + /** + * Merge any data from the Map stored in the URI-history store into the + * current instance. + * + * TODO: ensure compatibility with use of PersistLoadProcessor; suppress + * double-loading + * @param curi CrawlURI to receive prior state data + */ + @SuppressWarnings("unchecked") + protected void mergePrior(CrawlURI curi) { + Map prior = null; + String key = PersistProcessor.persistKeyFor(curi); + prior = (Map) store.get(key); + if(prior!=null) { + // merge in keys + curi.getData().putAll(prior); + } + } +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/QueuePrecedencePolicy.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/QueuePrecedencePolicy.java new file mode 100644 index 00000000..76282abb --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/QueuePrecedencePolicy.java @@ -0,0 +1,52 @@ +/* QueuePrecedencePolicy +* +* $Id: CostAssignmentPolicy.java 4981 2007-03-12 07:06:01Z paul_jack $ +* +* Created on Nov 17, 2007 +* +* Copyright (C) 2007 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.crawler.frontier.precedence; + +import java.io.Serializable; + +import org.archive.crawler.frontier.WorkQueue; + +/** + * Superclass for QueuePrecedencePolicies, which set a integer precedence value + * on uri-queues inside the frontier when the uri-queue is first created, and + * before the uri-queue is placed on a new internal queue-of-queues. + */ +abstract public class QueuePrecedencePolicy implements Serializable { + + /** + * Set an appropriate initial precedence value on the given + * newly-created WorkQueue. + * + * @param wq WorkQueue to modify + */ + abstract public void queueCreated(WorkQueue wq); + + /** + * Update an appropriate initial precedence value on the given + * already-existing WorkQueue. + * @param wq WorkQueue to modify + */ + abstract public void queueReevaluate(WorkQueue wq); +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/SimplePrecedenceProvider.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/SimplePrecedenceProvider.java new file mode 100644 index 00000000..0969a69a --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/SimplePrecedenceProvider.java @@ -0,0 +1,55 @@ +/* SimplePrecedenceProvider +* +* $Id: CostAssignmentPolicy.java 4981 2007-03-12 07:06:01Z paul_jack $ +* +* Created on Nov 20, 2007 +* +* Copyright (C) 2007 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.crawler.frontier.precedence; + +/** + * The most simple precedence provider, simply wrapping a resettable + * integer value. + */ +public class SimplePrecedenceProvider extends PrecedenceProvider { + private static final long serialVersionUID = 3132028374638763487L; + + int precedence; + + public SimplePrecedenceProvider(int p) { + this.precedence = p; + } + + /* (non-Javadoc) + * @see org.archive.crawler.frontier.PrecedenceProvider#getPrecedence() + */ + @Override + public int getPrecedence() { + return precedence; + } + + /** + * @param p + */ + public void setPrecedence(Integer p) { + precedence = p; + } + +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/SuccessCountsQueuePrecedencePolicy.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/SuccessCountsQueuePrecedencePolicy.java new file mode 100644 index 00000000..faa26f2f --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/SuccessCountsQueuePrecedencePolicy.java @@ -0,0 +1,112 @@ +/* SuccessCountsQueuePrecedencePolicy +* +* $Id: CostAssignmentPolicy.java 4981 2007-03-12 07:06:01Z paul_jack $ +* +* Created on Nov 17, 2007 +* +* Copyright (C) 2007 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.crawler.frontier.precedence; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.Transformer; +import org.archive.crawler.frontier.WorkQueue; + +/** + * QueuePrecedencePolicy that sets a uri-queue's precedence to a configured + * base value, then lowers its precedence with each tier of successful URIs + * completed. Any number of comma-separated tier sizes may be provided, with + * the last value assumed to repeat indefinitely. For example, with a + * 'base-precedence' value of 2, and 'increment-counts' of "100,1000", the + * queue will have a precedence of 2 until 100 URIs are successfully fetched, + * then a precedence of 3 for the next 1000 URIs successfully fetched, then + * continue to drop one precedence rank for each 1000 URIs successfully + * fetched. + */ +public class SuccessCountsQueuePrecedencePolicy extends BaseQueuePrecedencePolicy { + private static final long serialVersionUID = -4469760728466350850L; + +// TODO: determine why this doesn't work +// +// /** comma-separated list of success-counts at which precedence is bumped*/ +// final public static Key> INCREMENT_COUNTS = +// Key.make((List)Arrays.asList(new Integer[] {100})); +// +// /** +// * @param wq +// * @return +// */ +// protected int calculatePrecedence(WorkQueue wq) { +// // FIXME: it's inefficient to do this every time; optimizing +// // should be possible via more sophisticated custom PrecedenceProvider +// int precedence = wq.get(this,BASE_PRECEDENCE) - 1; +// Iterator iter = wq.get(this,INCREMENT_COUNTS).iterator(); +// int increment = iter.next(); +// long successes = wq.getSubstats().getFetchSuccesses(); +// while(successes>0) { +// successes -= increment; +// precedence++; +// increment = iter.hasNext() ? iter.next() : increment; +// } +// return precedence; +// } + + /** comma-separated list of success-counts at which precedence is bumped*/ + { + setIncrementCounts("100,1000"); + } + public String getIncrementCounts() { + return (String) kp.get("incrementCounts"); + } + public void setIncrementCounts(String counts) { + kp.put("incrementCounts",counts); + } + + /* (non-Javadoc) + * @see org.archive.crawler.frontier.QueuePrecedencePolicy#queueReevaluate(org.archive.crawler.frontier.WorkQueue) + */ + @SuppressWarnings("unchecked") + @Override + protected int calculatePrecedence(WorkQueue wq) { + // FIXME: it's ridiculously inefficient to do this every time, + // and optimizing will probably require inserting stateful policy + // helper object into WorkQueue -- expected when URI-precedence is + // also supported + int precedence = getBasePrecedence() - 1; + Collection increments = CollectionUtils.collect( + Arrays.asList(getIncrementCounts().split(",")), + new Transformer() { + public Object transform(final Object string) { + return Integer.parseInt((String)string); + }}); + Iterator iter = increments.iterator(); + int increment = iter.next(); + long successes = wq.getSubstats().getFetchSuccesses(); + while(successes>=0) { + successes -= increment; + precedence++; + increment = iter.hasNext() ? iter.next() : increment; + } + return precedence; + } +} diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/UriPrecedencePolicy.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/UriPrecedencePolicy.java new file mode 100644 index 00000000..53160397 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/UriPrecedencePolicy.java @@ -0,0 +1,51 @@ +/* UriPrecedencePolicy.java +* +* $Id: CostAssignmentPolicy.java 4981 2007-03-12 07:06:01Z paul_jack $ +* +* Created on Nov 18, 2007 +* +* Copyright (C) 2007 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.crawler.frontier.precedence; + +import java.io.Serializable; + +import org.archive.crawler.datamodel.CrawlURI; + +/** + * Superclass for URI precedence policies, which set a integer + * precedence value on individual URIs when they are first + * submitted to a frontier for scheduling. + * + * A URI's precedence directly affects where it lands in an + * individual URI queue, but does not affect a queue's precedence + * relative to other queues *unless* a queue-precedencence-policy + * that consults URI precedence values is chosen. + * + */ +abstract public class UriPrecedencePolicy implements Serializable { + + /** + * Add a precedence value to the supplied CrawlURI, which is being + * scheduled onto a frontier queue for the first time. + * @param curi CrawlURI to assign a precedence value + */ + abstract public void uriScheduled(CrawlURI curi); + +} diff --git a/engine/src/main/java/org/archive/crawler/io/NonFatalErrorFormatter.java b/engine/src/main/java/org/archive/crawler/io/NonFatalErrorFormatter.java new file mode 100644 index 00000000..bc7430e8 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/io/NonFatalErrorFormatter.java @@ -0,0 +1,53 @@ +/* Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * CrawlErrorFormatter.java + * Created on Jul 7, 2003 + * + * $Header$ + */ +package org.archive.crawler.io; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.logging.LogRecord; + +import org.archive.crawler.datamodel.CoreAttributeConstants; + +/** + * @author gojomo + * + */ +public class NonFatalErrorFormatter extends UriProcessingFormatter implements CoreAttributeConstants { + + /* (non-Javadoc) + * @see java.util.logging.Formatter#format(java.util.logging.LogRecord) + */ + public String format(LogRecord lr) { +// Throwable ex = lr.getThrown(); + Throwable ex = (Throwable)lr.getParameters()[1]; +// LocalizedError err = (LocalizedError) lr.getParameters()[1]; +// Throwable ex = (Throwable)err.exception; + StringWriter sw = new StringWriter(); + ex.printStackTrace(new PrintWriter(sw)); + + return super.format(lr) + " " + sw.toString(); + } +} + + diff --git a/engine/src/main/java/org/archive/crawler/io/RuntimeErrorFormatter.java b/engine/src/main/java/org/archive/crawler/io/RuntimeErrorFormatter.java new file mode 100644 index 00000000..de27b5c2 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/io/RuntimeErrorFormatter.java @@ -0,0 +1,64 @@ +/* RuntimeErrorFormatter + * + * Created on Jul 7, 2003 + * + * $Id$ + * + * Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.crawler.io; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.logging.LogRecord; + +import org.archive.crawler.datamodel.CoreAttributeConstants; +import org.archive.crawler.datamodel.CrawlURI; + +/** + * Runtime exception log formatter. + * + * Used to format unexpected runtime exceptions such as + * OOMEs. + * + * @author gojomo + */ +public class RuntimeErrorFormatter extends UriProcessingFormatter +implements CoreAttributeConstants { + public String format(LogRecord lr) { + Object [] parameters = lr.getParameters(); + String stackTrace = "None retrieved"; + if (parameters != null) { + // CrawlURI is always first parameter. + CrawlURI curi = (CrawlURI)parameters[0]; + if (curi != null) { + Throwable t = (Throwable)curi.getData().get(A_RUNTIME_EXCEPTION); + assert t != null : "Null throwable"; + StringWriter sw = new StringWriter(); + if (t == null) { + sw.write("No exception to report."); + } else { + t.printStackTrace(new PrintWriter(sw)); + } + stackTrace = sw.toString(); + } + } + return super.format(lr) + " " + stackTrace; + } +} diff --git a/engine/src/main/java/org/archive/crawler/io/StatisticsLogFormatter.java b/engine/src/main/java/org/archive/crawler/io/StatisticsLogFormatter.java new file mode 100644 index 00000000..139e2776 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/io/StatisticsLogFormatter.java @@ -0,0 +1,39 @@ +/* Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Created on Jul 22, 2003 + * + */ +package org.archive.crawler.io; + +import java.util.logging.Formatter; +import java.util.logging.LogRecord; + + +/** + * @author Parker Thompson + */ +public class StatisticsLogFormatter extends Formatter { + public StatisticsLogFormatter() { + super(); + } + + public String format(LogRecord record) { + return record.getMessage() + "\n"; + } +} diff --git a/engine/src/main/java/org/archive/crawler/io/UriErrorFormatter.java b/engine/src/main/java/org/archive/crawler/io/UriErrorFormatter.java new file mode 100644 index 00000000..29446771 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/io/UriErrorFormatter.java @@ -0,0 +1,59 @@ +/* Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * UriErrorFormatter.java + * Created on Jul 7, 2003 + * + * $Header$ + */ +package org.archive.crawler.io; + +import java.util.logging.Formatter; +import java.util.logging.LogRecord; + +import org.archive.crawler.datamodel.CoreAttributeConstants; +import org.archive.net.UURI; +import org.archive.util.ArchiveUtils; + +/** + * Formatter for 'uri-errors.log', of URIs so malformed they could + * not be instantiated. + * + * @author gojomo + * + */ +public class UriErrorFormatter extends Formatter implements CoreAttributeConstants { + + /* (non-Javadoc) + * @see java.util.logging.Formatter#format(java.util.logging.LogRecord) + */ + public String format(LogRecord lr) { + UURI uuri = (UURI) lr.getParameters()[0]; + String problem = (String) lr.getParameters()[1]; + + return ArchiveUtils.getLog17Date() + + " " + + ( (uuri ==null) ? "n/a" : uuri.toString() ) + + " \"" + + lr.getMessage() + + "\" " + + problem + + "\n"; + } +} + diff --git a/engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java b/engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java new file mode 100644 index 00000000..e4178da7 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java @@ -0,0 +1,149 @@ +/* UriProcessingFormatter.java + * + * $Id$ + * + * Created on Jun 10, 2003 + * + * Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.crawler.io; + +import it.unimi.dsi.mg4j.util.MutableString; + +import java.util.Collection; +import java.util.Iterator; +import java.util.logging.Formatter; +import java.util.logging.LogRecord; + +import org.archive.crawler.datamodel.CoreAttributeConstants; +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.util.ArchiveUtils; +import org.archive.util.MimetypeUtils; + +/** + * Formatter for 'crawl.log'. Expects completed CrawlURI as parameter. + * + * @author gojomo + */ +public class UriProcessingFormatter +extends Formatter implements CoreAttributeConstants { + private final static String NA = "-"; + /** + * Guess at line length (URIs are assumed avg. of 128 bytes). + * Used to preallocated the buffer we accumulate the log line + * in. Hopefully we get it right most of the time and no need + * to enlarge except in the rare case. + */ + private final static int GUESS_AT_LOG_LENGTH = + 17 + 1 + 3 + 1 + 10 + 128 + + 1 + 10 + 1 + 128 + 1 + 10 + 1 + 3 + + 14 + 1 + 32 + 4 + 128 + 1; + + /** + * Reuseable assembly buffer. + */ + private final MutableString buffer = + new MutableString(GUESS_AT_LOG_LENGTH); + + public String format(LogRecord lr) { + CrawlURI curi = (CrawlURI)lr.getParameters()[0]; + String length = NA; + String mime = null; + if (curi.isHttpTransaction()) { + if(curi.getContentLength() >= 0) { + length = Long.toString(curi.getContentLength()); + } else if (curi.getContentSize() > 0) { + length = Long.toString(curi.getContentSize()); + } + mime = curi.getContentType(); + } else { + if (curi.getContentSize() > 0) { + length = Long.toString(curi.getContentSize()); + } + mime = curi.getContentType(); + } + mime = MimetypeUtils.truncate(mime); + + long time = System.currentTimeMillis(); + String arcTimeAndDuration; + if(curi.containsDataKey(A_FETCH_COMPLETED_TIME)) { + long completedTime = curi.getFetchCompletedTime(); + long beganTime = curi.getFetchBeginTime(); + arcTimeAndDuration = ArchiveUtils.get17DigitDate(beganTime) + "+" + + Long.toString(completedTime - beganTime); + } else { + arcTimeAndDuration = NA; + } + + String via = curi.flattenVia(); + + String digest = curi.getContentDigestSchemeString(); + + String sourceTag = curi.containsDataKey(A_SOURCE_TAG) + ? curi.getSourceTag() + : null; + + this.buffer.length(0); + this.buffer.append(ArchiveUtils.getLog17Date(time)) + .append(" ") + .append(ArchiveUtils.padTo(curi.getFetchStatus(), 5)) + .append(" ") + .append(ArchiveUtils.padTo(length, 10)) + .append(" ") + .append(curi.getUURI().toString()) + .append(" ") + .append(checkForNull(curi.getPathFromSeed())) + .append(" ") + .append(checkForNull(via)) + .append(" ") + .append(mime) + .append(" ") + .append("#") + // Pad threads to be 3 digits. For Igor. + .append(ArchiveUtils.padTo( + Integer.toString(curi.getThreadNumber()), 3, '0')) + .append(" ") + .append(arcTimeAndDuration) + .append(" ") + .append(checkForNull(digest)) + .append(" ") + .append(checkForNull(sourceTag)) + .append(" "); + Collection anno = curi.getAnnotations(); + if ((anno != null) && (anno.size() > 0)) { + Iterator iter = anno.iterator(); + buffer.append(iter.next()); + while (iter.hasNext()) { + buffer.append(','); + buffer.append(iter.next()); + } + } + + return buffer.append("\n").toString(); + } + + /** + * @param str String to check. + * @return Return passed string or NA if null. + */ + protected String checkForNull(String str) { + return (str == null || str.length() <= 0)? NA: str; + } +} + + diff --git a/engine/src/main/java/org/archive/crawler/package.html b/engine/src/main/java/org/archive/crawler/package.html new file mode 100644 index 00000000..71b1a21a --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/package.html @@ -0,0 +1,162 @@ + + + +

Introduction to Heritrix.

+

+ Heritrix is designed to be easily extensible via 3rd party modules. + +

Architecture

+

+ The software is divided into several packages of varying importance. + The relationship between them will be covered in some greater depth + after their introductions. +

+ The root package (this) contains the executable class + {@link org.archive.crawler.Heritrix Heritrix}. + That class will load the crawler, parsing command line arguments. + If a WUI is to be launched it will launch it. It can also start + jobs (with or without the WUI) that are specified in command line + options. + +

framework

+

+ {@link org.archive.crawler.framework org.archive.crawler.framework} +

+ The framework package contains most of the core classes + for running a crawl. It also contains a number of Interfaces for + extensible items, the implementatations of whom can be found in + other classes. +

+ Heritrix is in effect divided into two types of classes. +

    +
  1. Core classes - these can often be configured but not + replaced. +
  2. Pluggable classes - these must implment a given interface + or extend a specific class but 3rd parties can introduce their own + implementations. +
+ The framework thus contains a selection of the core classes and a number + of the Interfaces and base classes for the pluggable classes. + +

datamodel

+

+ {@link org.archive.crawler.datamodel org.archive.crawler.datamodel} +

+ Contains various classes that make up the crawlers data structure. Including + such essentials as the CandidateURI and CrawlURI classes that wrap the + discovered URIs for processing. + +

admin

+

+ {@link org.archive.crawler.admin org.archive.crawler.admin} +

+ The admin package contains classes that are used by the Web UI. + This includes some core classes and a specific implementation of the + Statistics Tracking interface found in the framework + package that is designed to provide the UI with information about + ongoing crawls. + +

Pluggable modules

+

+ The following is a listing of the types of pluggable modules found in + Heritrix with brief explanations of each and linking to their respective + API documentation. + +

Frontier

+

+ A Frontier maintains the internal state of a crawl while it is + in progress. What URIs have been discovered, which should be crawled next, + etc. +

+ Needless to say this is one of the most important modules in any crawl and + the provided implementation should generally be appropriate unless a very + different strategy for ordering URIs for crawling is desired. +

+ {@link org.archive.crawler.framework.Frontier Frontier} is the interface + that all Frontiers must implement.
+ {@link org.archive.crawler.frontier org.archive.crawler.frontier} package + contains the provided implementation of a Frontier along with it's + supporting classes. + +

Processor

+Processing Steps +

+ When a URI is crawled, a {@link org.archive.crawler.framework.ToeThread + ToeThread} will execute a series of processors on it. +

+ The processors are split into 5 distinct chains that are exectued in sequence: + +

    +
  1. Pre-fetch processing chain +
  2. Fetch processing chain +
  3. Extractor processing chain +
  4. Write/Index processing chain +
  5. Post-processing chain +
+ Each of these chains contain any number of processors. The processors + all inherit from a generic {@link org.archive.crawler.framework.Processor + Processor}. While the processors are divided into the five categories above that + is strictly a high level configuration and any processor can be in any chain + (although doing link extraction before fetching a document is clearly of no + use). +

+ Numerous processors are provided with Heritrix in the following packages:
+ {@link org.archive.crawler.prefetch org.archive.crawler.prefetch} package + contains processors run before the URI is fetched from the Internet.
+ {@link org.archive.crawler.fetcher org.archive.crawler.fetcher} package + contains processors that fetch URI from the Internet. Typically each + processor handles a different protocol.
+ {@link org.archive.crawler.extractor org.archive.crawler.extractor} package + contains processors that perform link extractions on various document types.
+ {@link org.archive.crawler.writer org.archive.crawler.writer} package contains + a processor that writes an ARC file with the fetched document.
+ {@link org.archive.crawler.postprocessor org.archive.crawler.postprocessor} + package contain processors that do wrapup on the processing, reporting links + back to the Frontier etc. + +

Filter

+ +

Scope

+

+ Scopes are special filters that are applied to the crawl as a whole to + define it's scope. Any given crawl will employ exactly one scope + object to define what URIs are considered 'within scope'. +

+ Several implementations covering the most commonly + desired scopes are provided (broad, domain, host etc.). However custom + implementations can be made of these to define any arbitrary scope. + It should be noted though that usually any type of limitations to the scope + of a crawl can be more easily achived using one of the existing scopes and + modifing it with appropriate filters. +

+ {@link org.archive.crawler.framework.CrawlScope CrawlScope} - Base class for + scopes.
+ {@link org.archive.crawler.scope org.archive.crawler.scope} package. Contains + provided scopes. + +

Statistics Tracking

+

+ Any number of statistics tracking modules can be added to a crawl to gather + run time information about it's progress. +

+ These modules can both interrogate the Frontier for what sparse + date it exposes but they can also subscribe to + {@link org.archive.crawler.event.CrawlURIDispositionListener Crawled URI + Disposition} events to monitor the completion of each URI that is processed. +

+ An interface for {@link org.archive.crawler.framework.StatisticsTracking + statistics tracking} is provided as well as a partial implementation + ({@link org.archive.crawler.framework.AbstractTracker AbstractTracker}) + that does much of the work common to most statistics tracking modules. +

+ Furthermore the admin package implements a statistics tracking + module ({@link org.archive.crawler.admin.StatisticsTracker StatisticsTracker}) + that generates a log of the crawlers progress as well as providing information + that the UI uses. It also compiles end-of-crawl reports that contain all of the + information it has gathered in the course of the crawl.
+ It is highly recommended that it always be used when running crawls via the UI. + + + diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/CrawlStateUpdater.java b/engine/src/main/java/org/archive/crawler/postprocessor/CrawlStateUpdater.java new file mode 100644 index 00000000..ac9f7930 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/postprocessor/CrawlStateUpdater.java @@ -0,0 +1,117 @@ +/* CrawlStateUpdater + * + * Created on Jun 5, 2003 + * + * Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.crawler.postprocessor; + + +import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED; + +import java.util.logging.Logger; + +import org.apache.commons.httpclient.URIException; +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.PostProcessor; +import org.archive.modules.Processor; +import org.archive.modules.ProcessorURI; +import org.archive.modules.net.CrawlServer; +import org.archive.modules.net.RobotsHonoringPolicy; +import org.archive.modules.net.ServerCache; +import org.archive.modules.net.ServerCacheUtil; +import org.springframework.beans.factory.annotation.Autowired; + + +/** + * A step, late in the processing of a CrawlURI, for updating the per-host + * information that may have been affected by the fetch. This will initially + * be robots and ip address info; it could include other per-host stats that + * would affect the crawl (like total pages visited at the site) as well. + * + * @author gojomo + * @version $Date$, $Revision$ + */ +public class CrawlStateUpdater extends Processor implements + PostProcessor { + + private static final long serialVersionUID = -1072728147960180091L; + + private static final Logger logger = + Logger.getLogger(CrawlStateUpdater.class.getName()); + + + protected ServerCache serverCache; + public ServerCache getServerCache() { + return this.serverCache; + } + @Autowired + public void setServerCache(ServerCache serverCache) { + this.serverCache = serverCache; + } + + public RobotsHonoringPolicy getRobotsHonoringPolicy() { + return (RobotsHonoringPolicy) kp.get("robotsHonoringPolicy"); + } + @Autowired + public void setRobotsHonoringPolicy(RobotsHonoringPolicy policy) { + kp.put("robotsHonoringPolicy",policy); + } + + public CrawlStateUpdater() { + super(); + } + + @Override + protected boolean shouldProcess(ProcessorURI puri) { + return puri instanceof CrawlURI; + } + + @Override + protected void innerProcess(ProcessorURI puri) { + CrawlURI curi = (CrawlURI)puri; + + // Tally per-server, per-host, per-frontier-class running totals + CrawlServer server = ServerCacheUtil.getServerFor(serverCache, + curi.getUURI()); + + String scheme = curi.getUURI().getScheme().toLowerCase(); + if (scheme.equals("http") || scheme.equals("https") && + server != null) { + // Update connection problems counter + if(curi.getFetchStatus() == S_CONNECT_FAILED) { + server.incrementConsecutiveConnectionErrors(); + } else if (curi.getFetchStatus() > 0){ + server.resetConsecutiveConnectionErrors(); + } + + // Update robots info + try { + if (curi.getUURI().getPath() != null && + curi.getUURI().getPath().equals("/robots.txt")) { + // Update server with robots info + server.updateRobots(getRobotsHonoringPolicy(), curi); + } + } + catch (URIException e) { + logger.severe("Failed get path on " + curi.getUURI()); + } + } + } +} diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/FrontierScheduler.java b/engine/src/main/java/org/archive/crawler/postprocessor/FrontierScheduler.java new file mode 100644 index 00000000..5415031b --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/postprocessor/FrontierScheduler.java @@ -0,0 +1,107 @@ +/* FrontierScheduler + * + * $Id$ + * + * Created on June 6, 2005 + * + * Copyright (C) 2005 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ +package org.archive.crawler.postprocessor; + + +import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED; + +import java.util.concurrent.locks.ReentrantLock; + +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.crawler.framework.Frontier; +import org.archive.modules.PostProcessor; +import org.archive.modules.Processor; +import org.archive.modules.ProcessorURI; +import org.springframework.beans.factory.annotation.Autowired; + + +/** + * 'Schedule' with the Frontier CrawlURIs being carried by the passed + * CrawlURI. + * Adds either prerequisites or whatever is in CrawlURI outlinks to the + * Frontier. Run a Scoper ahead of this processor so only links that + * are in-scope get scheduled. + * @author stack + */ +public class FrontierScheduler extends Processor +implements PostProcessor { + + private static final long serialVersionUID = -3L; + + ReentrantLock lock = new ReentrantLock(true); + + /** + * The frontier to use. + */ + protected Frontier frontier; + public Frontier getFrontier() { + return this.frontier; + } + @Autowired + public void setFrontier(Frontier frontier) { + this.frontier = frontier; + } + + /** + */ + public FrontierScheduler() { + } + + protected boolean shouldProcess(ProcessorURI puri) { + return puri instanceof CrawlURI; + } + + @Override + protected void innerProcess(final ProcessorURI puri) { + CrawlURI curi = (CrawlURI)puri; + // Handle any prerequisites when S_DEFERRED for prereqs + if (curi.hasPrerequisiteUri() && curi.getFetchStatus() == S_DEFERRED) { + handlePrerequisites(curi); + return; + } + + try { + lock.lock(); + for (CrawlURI cauri: curi.getOutCandidates()) { + schedule(cauri); + } + } finally { + lock.unlock(); + } + } + + protected void handlePrerequisites(CrawlURI curi) { + schedule((CrawlURI)curi.getPrerequisiteUri()); + } + + /** + * Schedule the given {@link CrawlURI CrawlURI} with the Frontier. + * @param caUri The CrawlURI to be scheduled. + */ + protected void schedule(CrawlURI caUri) { + frontier.schedule(caUri); + } +} diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/LinksScoper.java b/engine/src/main/java/org/archive/crawler/postprocessor/LinksScoper.java new file mode 100644 index 00000000..db65aae2 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/postprocessor/LinksScoper.java @@ -0,0 +1,306 @@ +/* LinksScoper + * + * $Id$ + * + * Created on Oct 2, 2003 + * + * Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ +package org.archive.crawler.postprocessor; + +import static org.archive.crawler.datamodel.SchedulingConstants.HIGH; +import static org.archive.crawler.datamodel.SchedulingConstants.MEDIUM; +import static org.archive.crawler.datamodel.SchedulingConstants.NORMAL; +import static org.archive.modules.fetcher.FetchStatusCodes.S_PREREQUISITE_UNSCHEDULABLE_FAILURE; + +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.httpclient.URIException; +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.crawler.framework.Scoper; +import org.archive.modules.PostProcessor; +import org.archive.modules.ProcessorURI; +import org.archive.modules.deciderules.DecideResult; +import org.archive.modules.deciderules.DecideRule; +import org.archive.modules.deciderules.RejectDecideRule; +import org.archive.modules.extractor.Hop; +import org.archive.modules.extractor.Link; + +/** + * Determine which extracted links are within scope. + * TODO: To test scope, requires that Link be converted to + * a CrawlURI. Make it so don't have to make a CrawlURI to test + * if Link is in scope. + *

Since this scoper has to create CrawlURIs, no sense + * discarding them since later in the processing chain CrawlURIs rather + * than Links are whats needed scheduling extracted links w/ the + * Frontier (Frontier#schedule expects CrawlURI, not Link). This class + * replaces Links w/ the CrawlURI that wraps the Link in the CrawlURI. + * + * @author gojomo + * @author stack + */ +public class LinksScoper extends Scoper implements PostProcessor { + + private static final long serialVersionUID = -3L; + + private static Logger LOGGER = + Logger.getLogger(LinksScoper.class.getName()); + + /** + * If enabled, any URL found because a seed redirected to it (original seed + * returned 301 or 302), will also be treated as a seed. + */ + { + setSeedsRedirectNewSeeds(true); + } + public boolean getSeedsRedirectNewSeeds() { + return (Boolean) kp.get("seedsRedirectNewSeeds"); + } + public void setSeedsRedirectNewSeeds(boolean redirect) { + kp.put("seedsRedirectNewSeeds",redirect); + } + + /** + * DecideRules applied after an URI has been rejected. If the rules return + * {@link DecideResult#ACCEPT}, the URI is logged (if the logging level is + * INFO). Depends on {@link Scoper#OVERRIDE_LOGGER} being enabled. + */ + { + setLogRejectsRule(new RejectDecideRule()); + } + public DecideRule getLogRejectsRule() { + return (DecideRule) kp.get("logRejectsRule"); + } + public void setLogRejectsRule(DecideRule rule) { + kp.put("logRejectsRule", rule); + } + + /** + * Number of hops (of any sort) from a seed up to which a URI has higher + * priority scheduling than any remaining seed. For example, if set to 1 + * items one hop (link, embed, redirect, etc.) away from a seed will be + * scheduled with HIGH priority. If set to -1, no preferencing will occur, + * and a breadth-first search with seeds processed before discovered links + * will proceed. If set to zero, a purely depth-first search will proceed, + * with all discovered links processed before remaining seeds. Seed + * redirects are treated as one hop from a seed. + */ + { + setPreferenceDepthHops(-1); // no limit + } + public int getPreferenceDepthHops() { + return (Integer) kp.get("preferenceDepthHops"); + } + public void setPreferenceDepthHops(int depth) { + kp.put("preferenceDepthHops",depth); + } + + /** + * @param name Name of this filter. + */ + public LinksScoper() { + super(); + } + + + @Override + protected boolean shouldProcess(ProcessorURI puri) { + if (!(puri instanceof CrawlURI)) { + return false; + } + CrawlURI curi = (CrawlURI)puri; + + // If prerequisites, nothing to be done in here. + if (curi.hasPrerequisiteUri()) { + handlePrerequisite(curi); + return false; + } + + // Don't extract links of error pages. + if (curi.getFetchStatus() < 200 || curi.getFetchStatus() >= 400) { + curi.getOutLinks().clear(); + return false; + } + + if (curi.getOutLinks().isEmpty()) { + // No outlinks to process. + return false; + } + + return true; + } + + + @Override + protected void innerProcess(final ProcessorURI puri) { + CrawlURI curi = (CrawlURI)puri; + final boolean redirectsNewSeeds = getSeedsRedirectNewSeeds(); + int preferenceDepthHops = getPreferenceDepthHops(); + + for (Link wref: curi.getOutLinks()) try { + int directive = getSchedulingFor(curi, wref, preferenceDepthHops); + CrawlURI caURI = curi.createCrawlURI(curi.getBaseURI(), + wref, directive, + considerAsSeed(curi, wref, redirectsNewSeeds)); + if (isInScope(caURI)) { + curi.getOutCandidates().add(caURI); + } + } catch (URIException e) { + loggerModule.logUriError(e, curi.getUURI(), + wref.getDestination().toString()); + } + curi.getOutLinks().clear(); + +// Collection inScopeLinks = new HashSet(); +// for (final Iterator i = curi.getOutObjects().iterator(); i.hasNext();) { +// Object o = i.next(); +// if(o instanceof Link){ +// final Link wref = (Link)o; +// try { +// final int directive = getSchedulingFor(curi, wref, +// preferenceDepthHops); +// final CrawlURI caURI = +// curi.createCrawlURI(curi.getBaseURI(), wref, +// directive, +// considerAsSeed(curi, wref, redirectsNewSeeds)); +// if (isInScope(caURI)) { +// inScopeLinks.add(caURI); +// } +// } catch (URIException e) { +// getController().logUriError(e, curi.getUURI(), +// wref.getDestination().toString()); +// } +// } else if(o instanceof CrawlURI){ +// CrawlURI caURI = (CrawlURI)o; +// if(isInScope(caURI)){ +// inScopeLinks.add(caURI); +// } +// } else { +// LOGGER.severe("Unexpected type: " + o); +// } +// } +// // Replace current links collection w/ inscopeLinks. May be +// // an empty collection. +// curi.replaceOutlinks(inScopeLinks); + } + + /** + * The CrawlURI has a prerequisite; apply scoping and update + * Link to CrawlURI in manner analogous to outlink handling. + * @param curi CrawlURI with prereq to consider + */ + protected void handlePrerequisite(CrawlURI curi) { + try { + // Create prerequisite CrawlURI + CrawlURI caUri = + curi.createCrawlURI(curi.getBaseURI(), + (Link) curi.getPrerequisiteUri()); + int prereqPriority = curi.getSchedulingDirective() - 1; + if (prereqPriority < 0) { + prereqPriority = 0; + LOGGER.severe("Unable to promote prerequisite " + caUri + + " above " + curi); + } + caUri.setSchedulingDirective(prereqPriority); + caUri.setForceFetch(true); +// FIXME!!! getController().setStateProvider(caUri); + if(isInScope(caUri)) { + // replace link with CrawlURI + curi.setPrerequisiteUri(caUri); + } else { + // prerequisite is out-of-scope; mark CrawlURI as error, + // preventinting normal S_DEFERRED handling + curi.setFetchStatus(S_PREREQUISITE_UNSCHEDULABLE_FAILURE); + } + } catch (URIException ex) { + Object[] array = {curi, curi.getPrerequisiteUri()}; + loggerModule.getUriErrors().log(Level.INFO,ex.getMessage(), array); + } catch (NumberFormatException e) { + // UURI.createUURI will occasionally throw this error. + Object[] array = {curi, curi.getPrerequisiteUri()}; + loggerModule.getUriErrors().log(Level.INFO,e.getMessage(), array); + } + } + + protected void outOfScope(CrawlURI caUri) { + super.outOfScope(caUri); + if (!LOGGER.isLoggable(Level.INFO)) { + return; + } + DecideRule seq = getLogRejectsRule(); + if (seq.decisionFor(caUri) == DecideResult.ACCEPT) { + LOGGER.info(caUri.getUURI().toString()); + } + } + + private boolean considerAsSeed(final CrawlURI curi, final Link wref, + final boolean redirectsNewSeeds) { + // Check if this is a seed with a 301 or 302. + if (curi.isSeed() + && (curi.getFetchStatus() == 301 || + curi.getFetchStatus() == 302) + && wref.getHopType() == Hop.REFER) { + // Check if redirects from seeds should be treated as seeds. + if (redirectsNewSeeds) { + return true; + } + } + return false; + } + + /** + * Determine scheduling for the curi. + * As with the LinksScoper in general, this only handles extracted links, + * seeds do not pass through here, but are given MEDIUM priority. + * Imports into the frontier similarly do not pass through here, + * but are given NORMAL priority. + */ + protected int getSchedulingFor(final CrawlURI curi, final Link wref, + final int preferenceDepthHops) { + final Hop c = wref.getHopType(); + if (LOGGER.isLoggable(Level.FINEST)) { + LOGGER.finest(curi + " with path=" + curi.getPathFromSeed() + + " isSeed=" + curi.isSeed() + " with fetchStatus=" + + curi.getFetchStatus() + " -> " + wref.getDestination() + + " type " + c + " with context=" + wref.getContext()); + } + + switch (c) { + case REFER: + // Treat redirects somewhat urgently + // This also ensures seed redirects remain seed priority + return (preferenceDepthHops >= 0 ? HIGH : MEDIUM); + default: + if (preferenceDepthHops == 0) + return HIGH; + // this implies seed redirects are treated as path + // length 1, which I belive is standard. + // curi.getPathFromSeed() can never be null here, because + // we're processing a link extracted from curi + if (preferenceDepthHops > 0 && + curi.getPathFromSeed().length() + 1 <= preferenceDepthHops) + return HIGH; + // Everything else normal (at least for now) + return NORMAL; + } + } +} diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/LowDiskPauseProcessor.java b/engine/src/main/java/org/archive/crawler/postprocessor/LowDiskPauseProcessor.java new file mode 100644 index 00000000..ec4571b2 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/postprocessor/LowDiskPauseProcessor.java @@ -0,0 +1,178 @@ +/* + * LowDiskPauseProcessor + * + * $Id$ + * + * Created on Jun 5, 2003 + * + * Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.crawler.postprocessor; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.io.IOUtils; +import org.archive.modules.PostProcessor; +import org.archive.modules.ProcessResult; +import org.archive.modules.Processor; +import org.archive.modules.ProcessorURI; + +/** + * Processor module which uses 'df -k', where available and with + * the expected output format (on Linux), to monitor available + * disk space and pause the crawl if free space on monitored + * filesystems falls below certain thresholds. + */ +public class LowDiskPauseProcessor extends Processor implements PostProcessor { + + private static final long serialVersionUID = 3L; + + /** + * Logger. + */ + private static final Logger logger = + Logger.getLogger(LowDiskPauseProcessor.class.getName()); + + + /** + * List of filessystem mounts whose 'available' space should be monitored + * via 'df' (if available). + */ + List monitorMounts = new ArrayList(); + public List getMonitorMounts() { + return this.monitorMounts; + } + public void setMonitorMounts(List monitorMounts) { + this.monitorMounts = monitorMounts; + } + + /** + * When available space on any monitored mounts falls below this threshold, + * the crawl will be paused. + */ + int pauseThresholdKb = 500*1024; // 500MB + public int getPauseThresholdKb() { + return this.pauseThresholdKb; + } + public void setPauseThresholdKb(int pauseThresholdKb) { + this.pauseThresholdKb = pauseThresholdKb; + } + + /** + * Available space via 'df' is rechecked after every increment of this much + * content (uncompressed) is observed. + */ + int recheckThresholdKb = 200*1024; // 200MB + public int getRecheckThresholdKb() { + return this.recheckThresholdKb; + } + public void setRecheckThresholdKb(int recheckThresholdKb) { + this.recheckThresholdKb = recheckThresholdKb; + } + + protected int contentSinceCheck = 0; + + public static final Pattern VALID_DF_OUTPUT = + Pattern.compile("(?s)^Filesystem\\s+1K-blocks\\s+Used\\s+Available\\s+Use%\\s+Mounted on\\n.*"); + public static final Pattern AVAILABLE_EXTRACTOR = + Pattern.compile("(?m)\\s(\\d+)\\s+\\d+%\\s+(\\S+)$"); + + /** + * @param name Name of this writer. + */ + public LowDiskPauseProcessor() { + } + + + @Override + protected boolean shouldProcess(ProcessorURI curi) { + return true; + } + + @Override + protected void innerProcess(ProcessorURI uri) { + throw new AssertionError(); + } + + /** + * Notes a CrawlURI's content size in its running tally. If the + * recheck increment of content has passed through since the last + * available-space check, checks available space and pauses the + * crawl if any monitored mounts are below the configured threshold. + * + * @param curi CrawlURI to process. + */ + @Override + protected ProcessResult innerProcessResult(ProcessorURI curi) { + synchronized (this) { + contentSinceCheck += curi.getContentSize(); + if (contentSinceCheck/1024 > getRecheckThresholdKb()) { + ProcessResult r = checkAvailableSpace(curi); + contentSinceCheck = 0; + return r; + } else { + return ProcessResult.PROCEED; + } + } + } + + + /** + * Probe via 'df' to see if monitored mounts have fallen + * below the pause available threshold. If so, request a + * crawl pause. + * @param curi Current context. + */ + private ProcessResult checkAvailableSpace(ProcessorURI curi) { + try { + String df = IOUtils.toString(Runtime.getRuntime().exec( + "df -k").getInputStream()); + Matcher matcher = VALID_DF_OUTPUT.matcher(df); + if(!matcher.matches()) { + logger.severe("'df -k' output unacceptable for low-disk checking"); + return ProcessResult.PROCEED; + } + List monitoredMounts = getMonitorMounts(); + matcher = AVAILABLE_EXTRACTOR.matcher(df); + while (matcher.find()) { + String mount = matcher.group(2); + if (monitoredMounts.contains(mount)) { + long availKilobytes = Long.parseLong(matcher.group(1)); + int thresholdKilobytes = getPauseThresholdKb(); + if (availKilobytes < thresholdKilobytes ) { + logger.log(Level.SEVERE, "Low Disk Pause", + availKilobytes + "K available on " + mount + + " (below threshold " + + thresholdKilobytes + "K)"); + return ProcessResult.STUCK; + } + } + } + } catch (IOException e) { + curi.getNonFatalFailures().add(e); + } + return ProcessResult.PROCEED; + } +} diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java b/engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java new file mode 100644 index 00000000..f24c35db --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java @@ -0,0 +1,142 @@ +/* SupplementaryLinksScoper + * + * $Id$ + * + * Created on Oct 2, 2003 + * + * Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ +package org.archive.crawler.postprocessor; + +import java.util.Iterator; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.crawler.framework.Scoper; +import org.archive.modules.PostProcessor; +import org.archive.modules.ProcessorURI; +import org.archive.modules.deciderules.AcceptDecideRule; +import org.archive.modules.deciderules.DecideResult; +import org.archive.modules.deciderules.DecideRule; + + +/** + * Run CrawlURI links carried in the passed CrawlURI through a filter + * and 'handle' rejections. + * Used to do supplementary processing of links after they've been scope + * processed and ruled 'in-scope' by LinkScoper. An example of + * 'supplementary processing' would check that a Link is intended for + * this host to crawl in a multimachine crawl setting. Configure filters to + * rule on links. Default handler writes rejected URLs to disk. Subclass + * to handle rejected URLs otherwise. + * @author stack + */ +public class SupplementaryLinksScoper extends Scoper implements PostProcessor { + + private static final long serialVersionUID = -3L; + + private static Logger LOGGER = + Logger.getLogger(SupplementaryLinksScoper.class.getName()); + + + /** + * DecideRules which if their final decision on a link is + * REJECT, cause the link to be ruled out-of-scope, even + * if it had previously been accepted by the main scope. + */ + { + setSupplementaryRule(new AcceptDecideRule()); + } + public DecideRule getSupplementaryRule() { + return (DecideRule) kp.get("supplementaryRule"); + } + public void setSupplementaryRule(DecideRule rule) { + kp.put("supplementaryRule", rule); + } + + /** + * @param name Name of this filter. + */ + public SupplementaryLinksScoper() { + super(); + } + + + protected boolean shouldProcess(ProcessorURI puri) { + return puri instanceof CrawlURI; + } + + + protected void innerProcess(final ProcessorURI puri) { + CrawlURI curi = (CrawlURI)puri; + + // If prerequisites or no links, nothing to be done in here. + if (curi.hasPrerequisiteUri() || curi.getOutLinks().isEmpty()) { + return; + } + +// Collection inScopeLinks = new HashSet(); + Iterator iter = curi.getOutCandidates().iterator(); + while (iter.hasNext()) { + CrawlURI cauri = iter.next(); + if (!isInScope(cauri)) { + iter.remove(); + } + } +// for (CrawlURI cauri: curi.getOutCandidates()) { +// if (isInScope(cauri)) { +// inScopeLinks.add(cauri); +// } +// } + // Replace current links collection w/ inscopeLinks. May be + // an empty collection. +// curi.replaceOutlinks(inScopeLinks); + } + + protected boolean isInScope(CrawlURI caUri) { + // TODO: Fix filters so work on CrawlURI. + CrawlURI curi = (caUri instanceof CrawlURI)? + (CrawlURI)caUri: + new CrawlURI(caUri.getUURI()); + boolean result = false; + DecideRule seq = getSupplementaryRule(); + if (seq.decisionFor(curi) == DecideResult.ACCEPT) { + result = true; + if (LOGGER.isLoggable(Level.FINER)) { + LOGGER.finer("Accepted: " + caUri); + } + } else { + outOfScope(caUri); + } + return result; + } + + /** + * Called when a CrawlURI is ruled out of scope. + * @param caUri CrawlURI that is out of scope. + */ + protected void outOfScope(CrawlURI caUri) { + if (!LOGGER.isLoggable(Level.INFO)) { + return; + } + LOGGER.info(caUri.getUURI().toString()); + } +} diff --git a/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java b/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java new file mode 100644 index 00000000..68008c75 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java @@ -0,0 +1,529 @@ +/* Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * SimplePolitenessEnforcer.java + * Created on May 22, 2003 + * + * $Header$ + */ +package org.archive.crawler.prefetch; + +import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED; +import static org.archive.modules.fetcher.FetchStatusCodes.S_DOMAIN_PREREQUISITE_FAILURE; +import static org.archive.modules.fetcher.FetchStatusCodes.S_ROBOTS_PRECLUDED; +import static org.archive.modules.fetcher.FetchStatusCodes.S_ROBOTS_PREREQUISITE_FAILURE; +import static org.archive.modules.fetcher.FetchStatusCodes.S_UNFETCHABLE_URI; + +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.httpclient.URIException; +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.crawler.reporting.CrawlerLoggerModule; +import org.archive.modules.ProcessResult; +import org.archive.modules.Processor; +import org.archive.modules.ProcessorURI; +import org.archive.modules.credential.Credential; +import org.archive.modules.credential.CredentialAvatar; +import org.archive.modules.credential.CredentialStore; +import org.archive.modules.extractor.Hop; +import org.archive.modules.extractor.Link; +import org.archive.modules.extractor.LinkContext; +import org.archive.modules.fetcher.UserAgentProvider; +import org.archive.modules.net.CrawlHost; +import org.archive.modules.net.CrawlServer; +import org.archive.modules.net.ServerCache; +import org.archive.modules.net.ServerCacheUtil; +import org.archive.net.UURI; +import org.archive.net.UURIFactory; +import org.springframework.beans.factory.annotation.Autowired; + + +/** + * Ensures the preconditions for a fetch -- such as DNS lookup + * or acquiring and respecting a robots.txt policy -- are + * satisfied before a URI is passed to subsequent stages. + * + * @author gojomo + */ +public class PreconditionEnforcer extends Processor { + + private static final long serialVersionUID = 3L; + + private static final Logger logger = + Logger.getLogger(PreconditionEnforcer.class.getName()); + + + public UserAgentProvider getUserAgentProvider() { + return (UserAgentProvider) kp.get("userAgentProvider"); + } + @Autowired + public void setUserAgentProvider(UserAgentProvider provider) { + kp.put("userAgentProvider",provider); + } + + /** + * The minimum interval for which a dns-record will be considered + * valid (in seconds). If the record's DNS TTL is larger, that will + * be used instead. + */ + { + setIpValidityDurationSeconds(6*60*60); // 6 hours + } + public int getIpValidityDurationSeconds() { + return (Integer) kp.get("ipValidityDurationSeconds"); + } + public void setIpValidityDurationSeconds(int duration) { + kp.put("ipValidityDurationSeconds",duration); + } + + /** + * The time in seconds that fetched robots.txt information is considered to + * be valid. If the value is set to '0', then the robots.txt information + * will never expire. + */ + { + setRobotsValidityDurationSeconds(24*60*60); // 24 hours + } + public int getRobotsValidityDurationSeconds() { + return (Integer) kp.get("robotsValidityDurationSeconds"); + } + public void setRobotsValidityDurationSeconds(int duration) { + kp.put("robotsValidityDurationSeconds",duration); + } + + /** + * Whether to only calculate the robots status of an URI, without actually + * applying any exclusions found. If true, exlcuded URIs will only be + * annotated in the crawl.log, but still fetched. Default is false. + */ + { + setCalculateRobotsOnly(false); + } + public boolean getCalculateRobotsOnly() { + return (Boolean) kp.get("calculateRobotsOnly"); + } + public void setCalculateRobotsOnly(boolean recheck) { + kp.put("calculateRobotsOnly",recheck); + } + + protected ServerCache serverCache; + public ServerCache getServerCache() { + return this.serverCache; + } + @Autowired + public void setServerCache(ServerCache serverCache) { + this.serverCache = serverCache; + } + + public CredentialStore getCredentialStore() { + return (CredentialStore) kp.get("credentialStore"); + } + @Autowired + public void setCredentialStore(CredentialStore credentials) { + kp.put("credentialStore",credentials); + } + + protected CrawlerLoggerModule loggerModule; + public CrawlerLoggerModule getLoggerModule() { + return this.loggerModule; + } + @Autowired + public void setLoggerModule(CrawlerLoggerModule loggerModule) { + this.loggerModule = loggerModule; + } + + public PreconditionEnforcer() { + super(); + } + + @Override + protected boolean shouldProcess(ProcessorURI puri) { + return (puri instanceof CrawlURI); + } + + + @Override + protected void innerProcess(ProcessorURI puri) { + throw new AssertionError(); + } + + + @Override + protected ProcessResult innerProcessResult(ProcessorURI puri) { + CrawlURI curi = (CrawlURI)puri; + if (considerDnsPreconditions(curi)) { + return ProcessResult.FINISH; + } + + // make sure we only process schemes we understand (i.e. not dns) + String scheme = curi.getUURI().getScheme().toLowerCase(); + if (! (scheme.equals("http") || scheme.equals("https"))) { + logger.fine("PolitenessEnforcer doesn't understand uri's of type " + + scheme + " (ignoring)"); + return ProcessResult.PROCEED; + } + + if (considerRobotsPreconditions(curi)) { + return ProcessResult.FINISH; + } + + if (!curi.isPrerequisite() && credentialPrecondition(curi)) { + return ProcessResult.FINISH; + } + + // OK, it's allowed + + // For all curis that will in fact be fetched, set appropriate delays. + // TODO: SOMEDAY: allow per-host, per-protocol, etc. factors + // curi.setDelayFactor(getDelayFactorFor(curi)); + // curi.setMinimumDelay(getMinimumDelayFor(curi)); + + return ProcessResult.PROCEED; + } + + /** + * Consider the robots precondition. + * + * @param curi ProcessorURI we're checking for any required preconditions. + * @return True, if this curi has a precondition or processing + * should be terminated for some other reason. False if + * we can precede to process this url. + */ + private boolean considerRobotsPreconditions(CrawlURI curi) { + // treat /robots.txt fetches specially + UURI uuri = curi.getUURI(); + try { + if (uuri != null && uuri.getPath() != null && + curi.getUURI().getPath().equals("/robots.txt")) { + // allow processing to continue + curi.setPrerequisite(true); + return false; + } + } + catch (URIException e) { + logger.severe("Failed get of path for " + curi); + } + + // require /robots.txt if not present + if (isRobotsExpired(curi)) { + // Need to get robots + if (logger.isLoggable(Level.FINE)) { + CrawlServer server = getServerFor(curi); + logger.fine( "No valid robots for " + server + + "; deferring " + curi); + } + + // Robots expired - should be refetched even though its already + // crawled. + try { + String prereq = curi.getUURI().resolve("/robots.txt").toString(); + markPrerequisite(curi, prereq); + } + catch (URIException e1) { + logger.severe("Failed resolve using " + curi); + throw new RuntimeException(e1); // shouldn't ever happen + } + return true; + } + // test against robots.txt if available + CrawlServer cs = getServerFor(curi); + if (cs.isValidRobots()) { + String ua = getUserAgentProvider().getUserAgent(); + if(cs.getRobots().disallows(curi, ua)) { + if(getCalculateRobotsOnly()) { + // annotate URI as excluded, but continue to process normally + curi.getAnnotations().add("robotExcluded"); + return false; + } + // mark as precluded; in FetchHTTP, this will + // prevent fetching and cause a skip to the end + // of processing (unless an intervening processor + // overrules) + curi.setFetchStatus(S_ROBOTS_PRECLUDED); + curi.setError("robots.txt exclusion"); + logger.fine("robots.txt precluded " + curi); + return true; + } + return false; + } + // No valid robots found => Attempt to get robots.txt failed +// curi.skipToPostProcessing(); + curi.setFetchStatus(S_ROBOTS_PREREQUISITE_FAILURE); + curi.setError("robots.txt prerequisite failed"); + if (logger.isLoggable(Level.FINE)) { + logger.fine("robots.txt prerequisite failed " + curi); + } + return true; + } + + /** + * @param curi ProcessorURI whose dns prerequisite we're to check. + * @return true if no further processing in this module should occur + */ + private boolean considerDnsPreconditions(CrawlURI curi) { + if(curi.getUURI().getScheme().equals("dns")){ + // DNS URIs never have a DNS precondition + curi.setPrerequisite(true); + return false; + } + + CrawlServer cs = getServerFor(curi); + if(cs == null) { + curi.setFetchStatus(S_UNFETCHABLE_URI); +// curi.skipToPostProcessing(); + return true; + } + + // If we've done a dns lookup and it didn't resolve a host + // cancel further fetch-processing of this URI, because + // the domain is unresolvable + CrawlHost ch = getHostFor(curi); + if (ch == null || ch.hasBeenLookedUp() && ch.getIP() == null) { + if (logger.isLoggable(Level.FINE)) { + logger.fine( "no dns for " + ch + + " cancelling processing for ProcessorURI " + curi.toString()); + } + curi.setFetchStatus(S_DOMAIN_PREREQUISITE_FAILURE); +// curi.skipToPostProcessing(); + return true; + } + + // If we haven't done a dns lookup and this isn't a dns uri + // shoot that off and defer further processing + if (isIpExpired(curi) && !curi.getUURI().getScheme().equals("dns")) { + logger.fine("Deferring processing of ProcessorURI " + curi.toString() + + " for dns lookup."); + String preq = "dns:" + ch.getHostName(); + try { + markPrerequisite(curi, preq); + } catch (URIException e) { + throw new RuntimeException(e); // shouldn't ever happen + } + return true; + } + + // DNS preconditions OK + return false; + } + + /** Return true if ip should be looked up. + * + * @param curi the URI to check. + * @return true if ip should be looked up. + */ + public boolean isIpExpired(ProcessorURI curi) { + CrawlHost host = getHostFor(curi); + if (!host.hasBeenLookedUp()) { + // IP has not been looked up yet. + return true; + } + + if (host.getIpTTL() == CrawlHost.IP_NEVER_EXPIRES) { + // IP never expires (numeric IP) + return false; + } + + long duration = getIpValidityDurationSeconds(); + if (duration == 0) { + // Never expire ip if duration is null (set by user or more likely, + // set to zero in case where we tried in FetchDNS but failed). + return false; + } + + long ttl = host.getIpTTL(); + if (ttl > duration) { + // Use the larger of the operator-set minimum duration + // or the DNS record TTL + duration = ttl; + } + + // Duration and ttl are in seconds. Convert to millis. + if (duration > 0) { + duration *= 1000; + } + + return (duration + host.getIpFetched()) < System.currentTimeMillis(); + } + + /** + * Is the robots policy expired. + * + * This method will also return true if we haven't tried to get the + * robots.txt for this server. + * + * @param curi + * @return true if the robots policy is expired. + */ + public boolean isRobotsExpired(ProcessorURI curi) { + CrawlServer server = getServerFor(curi); + long robotsFetched = server.getRobotsFetchedTime(); + if (robotsFetched == CrawlServer.ROBOTS_NOT_FETCHED) { + // Have not attempted to fetch robots + return true; + } + long duration = getRobotsValidityDurationSeconds()*1000L; + if (duration == 0) { + // When zero, robots should be valid forever + return false; + } + if (robotsFetched + duration < System.currentTimeMillis()) { + // Robots is still valid + return true; + } + return false; + } + + /** + * Consider credential preconditions. + * + * Looks to see if any credential preconditions (e.g. html form login + * credentials) for this CrawlServer. If there are, have they + * been run already? If not, make the running of these logins a precondition + * of accessing any other url on this CrawlServer. + * + *

+ * One day, do optimization and avoid running the bulk of the code below. + * Argument for running the code everytime is that overrides and refinements + * may change what comes back from credential store. + * + * @param curi ProcessorURI we're checking for any required preconditions. + * @return True, if this curi has a precondition that needs to + * be met before we can proceed. False if we can precede to process + * this url. + */ + private boolean credentialPrecondition(final CrawlURI curi) { + + boolean result = false; + + CredentialStore cs = getCredentialStore(); + if (cs == null) { + logger.severe("No credential store for " + curi); + return result; + } + + for (Credential c: cs.getAll()) { + if (c.isPrerequisite(curi)) { + // This credential has a prereq. and this curi is it. Let it + // through. Add its avatar to the curi as a mark. Also, does + // this curi need to be posted? Note, we do this test for + // is it a prereq BEFORE we do the check that curi is of the + // credential domain because such as yahoo have you go to + // another domain altogether to login. + c.attach(curi); + curi.setFetchType(ProcessorURI.FetchType.HTTP_POST); + break; + } + + if (!c.rootUriMatch(serverCache, curi)) { + continue; + } + + if (!c.hasPrerequisite(curi)) { + continue; + } + + if (!authenticated(c, curi)) { + // Han't been authenticated. Queue it and move on (Assumption + // is that we can do one authentication at a time -- usually one + // html form). + String prereq = c.getPrerequisite(curi); + if (prereq == null || prereq.length() <= 0) { + CrawlServer server = getServerFor(curi); + logger.severe(server.getName() + " has " + + " credential(s) of type " + c + " but prereq" + + " is null."); + } else { + try { + markPrerequisite(curi, prereq); + } catch (URIException e) { + logger.severe("unable to set credentials prerequisite "+prereq); + loggerModule.logUriError(e,curi.getUURI(),prereq); + return false; + } + result = true; + if (logger.isLoggable(Level.FINE)) { + logger.fine("Queueing prereq " + prereq + " of type " + + c + " for " + curi); + } + break; + } + } + } + return result; + } + + /** + * Has passed credential already been authenticated. + * + * @param credential Credential to test. + * @param curi ProcessorURI. + * @return True if already run. + */ + private boolean authenticated(final Credential credential, + final ProcessorURI curi) { + boolean result = false; + CrawlServer server = getServerFor(curi); + if (!server.hasCredentialAvatars()) { + return result; + } + Set avatars = server.getCredentialAvatars(); + for (CredentialAvatar ca: avatars) { + String key = null; + key = credential.getKey(); + if (ca.match(credential.getClass(), key)) { + result = true; + } + } + return result; + } + + + /** + * Do all actions associated with setting a CrawlURI as + * requiring a prerequisite. + * + * @param lastProcessorChain Last processor chain reference. This chain is + * where this CrawlURI goes next. + * @param preq Object to set a prerequisite. + * @throws URIException + */ + private void markPrerequisite(CrawlURI curi, String preq) + throws URIException { + UURI src = curi.getUURI(); + UURI dest = UURIFactory.getInstance(preq); + LinkContext lc = LinkContext.PREREQ_MISC; + Hop hop = Hop.PREREQ; + Link link = new Link(src, dest, lc, hop); + curi.setPrerequisiteUri(link); + curi.incrementDeferrals(); + curi.setFetchStatus(S_DEFERRED); + //skipToPostProcessing(); + } + + + private CrawlServer getServerFor(ProcessorURI curi) { + return ServerCacheUtil.getServerFor(serverCache, curi.getUURI()); + } + + + private CrawlHost getHostFor(ProcessorURI curi) { + return ServerCacheUtil.getHostFor(serverCache, curi.getUURI()); + } +} diff --git a/engine/src/main/java/org/archive/crawler/prefetch/Preselector.java b/engine/src/main/java/org/archive/crawler/prefetch/Preselector.java new file mode 100644 index 00000000..f2b3b992 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/prefetch/Preselector.java @@ -0,0 +1,165 @@ +/* Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * SimplePreselector.java + * Created on Sep 22, 2003 + * + * $Header$ + */ +package org.archive.crawler.prefetch; + + +import org.archive.crawler.datamodel.CrawlURI; + +import static org.archive.modules.fetcher.FetchStatusCodes.*; + +import org.archive.crawler.framework.Scoper; +import org.archive.modules.ProcessResult; +import org.archive.modules.ProcessorURI; +import org.archive.util.TextUtils; + + +/** + * If set to recheck the crawl's scope, gives a yes/no on whether + * a CrawlURI should be processed at all. If not, its status + * will be marked OUT_OF_SCOPE and the URI will skip directly + * to the first "postprocessor". + * + * + * @author gojomo + * + */ +public class Preselector extends Scoper { + private static final long serialVersionUID = 3L; + + /** + * Recheck if uri is in scope. This is meaningful if the scope is altered + * during a crawl. URIs are checked against the scope when they are added to + * queues. Setting this value to true forces the URI to be checked against + * the scope when it is comming out of the queue, possibly after the scope + * is altered. + */ + { + setRecheckScope(false); + } + public boolean getRecheckScope() { + return (Boolean) kp.get("recheckScope"); + } + public void setRecheckScope(boolean recheck) { + kp.put("recheckScope",recheck); + } + + /** + * Block all URIs from being processed. This is most likely to be used in + * overrides to easily reject certain hosts from being processed. + */ + { + setBlockAll(false); + } + public boolean getBlockAll() { + return (Boolean) kp.get("blockAll"); + } + public void setBlockAll(boolean recheck) { + kp.put("blockAll",recheck); + } + + /** + * Block all URIs matching the regular expression from being processed. + */ + { + setBlockByRegex(""); + } + public String getBlockByRegex() { + return (String) kp.get("blockByRegex"); + } + public void setBlockByRegex(String regex) { + kp.put("blockByRegex",regex); + } + + /** + * Allow only URIs matching the regular expression to be processed. + */ + { + setAllowByRegex(""); + } + public String getAllowByRegex() { + return (String) kp.get("allowByRegex"); + } + public void setAllowByRegex(String regex) { + kp.put("allowByRegex",regex); + } + + /** + * Constructor. + */ + public Preselector() { + super(); + } + + @Override + protected boolean shouldProcess(ProcessorURI puri) { + return puri instanceof CrawlURI; + } + + + @Override + protected void innerProcess(ProcessorURI puri) { + throw new AssertionError(); + } + + + @Override + protected ProcessResult innerProcessResult(ProcessorURI puri) { + CrawlURI curi = (CrawlURI)puri; + + // Check if uris should be blocked + if (getBlockAll()) { + curi.setFetchStatus(S_BLOCKED_BY_USER); + return ProcessResult.FINISH; + } + + // Check if allowed by regular expression + String regexp = getAllowByRegex(); + if (regexp != null && !regexp.equals("")) { + if (!TextUtils.matches(regexp, curi.toString())) { + curi.setFetchStatus(S_BLOCKED_BY_USER); + return ProcessResult.FINISH; + } + } + + // Check if blocked by regular expression + regexp = getBlockByRegex(); + if (regexp != null && !regexp.equals("")) { + if (TextUtils.matches(regexp, curi.toString())) { + curi.setFetchStatus(S_BLOCKED_BY_USER); + return ProcessResult.FINISH; + } + } + + // Possibly recheck scope + if (getRecheckScope()) { + if (!isInScope(curi)) { + // Scope rejected + curi.setFetchStatus(S_OUT_OF_SCOPE); + return ProcessResult.FINISH; + } + } + + return ProcessResult.PROCEED; + } +} diff --git a/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java b/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java new file mode 100644 index 00000000..0beaf41e --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java @@ -0,0 +1,400 @@ +/* QuotaEnforcer + * + * Created on Nov 4, 2005 + * + * Copyright (C) 2005 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.crawler.prefetch; + +import static org.archive.modules.fetcher.FetchStatusCodes.S_BLOCKED_BY_QUOTA; + +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.crawler.framework.Frontier; +import org.archive.modules.ProcessResult; +import org.archive.modules.Processor; +import org.archive.modules.ProcessorURI; +import org.archive.modules.fetcher.FetchStats; +import org.archive.modules.net.CrawlHost; +import org.archive.modules.net.CrawlServer; +import org.archive.modules.net.ServerCache; +import org.archive.modules.net.ServerCacheUtil; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * A simple quota enforcer. If the host, server, or frontier group + * associated with the current CrawlURI is already over its quotas, + * blocks the current URI's processing with S_BLOCKED_BY_QUOTA. + * + * @author gojomo + * @version $Date$, $Revision$ + */ +public class QuotaEnforcer extends Processor { + private static final long serialVersionUID = 3L; + + private static final Logger LOGGER = + Logger.getLogger(QuotaEnforcer.class.getName()); + + // indexed table of reused string categorical names/keys + protected static final int SERVER = 0; + protected static final int HOST = 1; + protected static final int GROUP = 2; + + protected static final int SUCCESSES = 0; + protected static final int SUCCESS_KB = 1; + protected static final int RESPONSES = 2; + protected static final int RESPONSE_KB = 3; + + private static final String SERVER_MAX_FETCH_SUCCESSES = "serverMaxFetchSuccesses"; + private static final String SERVER_MAX_SUCCESS_KB = "serverMaxSuccessKb"; + private static final String SERVER_MAX_FETCH_RESPONSES = "serverMaxFetchResponses"; + private static final String SERVER_MAX_ALL_KB = "serverMaxAllKb"; + + private static final String HOST_MAX_FETCH_SUCCESSES = "hostMaxFetchSuccesses"; + private static final String HOST_MAX_SUCCESS_KB = "hostMaxSuccessKb"; + private static final String HOST_MAX_FETCH_RESPONSES = "hostMaxFetchResponses"; + private static final String HOST_MAX_ALL_KB = "hostMaxAllKb"; + + private static final String GROUP_MAX_FETCH_SUCCESSES = "groupMaxFetchSuccesses"; + private static final String GROUP_MAX_SUCCESS_KB = "groupMaxSuccessKb"; + private static final String GROUP_MAX_FETCH_RESPONSES = "groupMaxFetchResponses"; + private static final String GROUP_MAX_ALL_KB = "groupMaxAllKb"; + + protected static final String[][] keys = new String[][] { + { + //"server", + SERVER_MAX_FETCH_SUCCESSES, + SERVER_MAX_SUCCESS_KB, + SERVER_MAX_FETCH_RESPONSES, + SERVER_MAX_ALL_KB + }, + { + //"host" + HOST_MAX_FETCH_SUCCESSES, + HOST_MAX_SUCCESS_KB, + HOST_MAX_FETCH_RESPONSES, + HOST_MAX_ALL_KB + , + }, + { + //"group" + GROUP_MAX_FETCH_SUCCESSES, + GROUP_MAX_SUCCESS_KB, + GROUP_MAX_FETCH_RESPONSES, + GROUP_MAX_ALL_KB + } + }; + + // server quotas + // successes + + /** + * Maximum number of fetch successes (e.g. 200 responses) to collect from + * one server. Default is -1, meaning no limit. + */ + { + setServerMaxFetchSuccesses(-1L); // no limit + } + public long getServerMaxFetchSuccesses() { + return (Long) kp.get(SERVER_MAX_FETCH_SUCCESSES); + } + public void setServerMaxFetchSuccesses(long max) { + kp.put(SERVER_MAX_FETCH_SUCCESSES,max); + } + + + /** + * Maximum amount of fetch success content (e.g. 200 responses) in KB to + * collect from one server. Default is -1, meaning no limit. + */ + { + setServerMaxSuccessKb(-1L); // no limit + } + public long getServerMaxSuccessKb() { + return (Long) kp.get(SERVER_MAX_SUCCESS_KB); + } + public void setServerMaxSuccessKb(long max) { + kp.put(SERVER_MAX_SUCCESS_KB,max); + } + + /** + * Maximum number of fetch responses (incl. error responses) to collect from + * one server. Default is -1, meaning no limit. + */ + { + setServerMaxFetchResponses(-1L); // no limit + } + public long getServerMaxFetchResponses() { + return (Long) kp.get(SERVER_MAX_FETCH_RESPONSES); + } + public void setServerMaxFetchResponses(long max) { + kp.put(SERVER_MAX_FETCH_RESPONSES,max); + } + + /** + * Maximum amount of response content (incl. error responses) in KB to + * collect from one server. Default is -1, meaning no limit. + */ + { + setServerMaxAllKb(-1L); // no limit + } + public long getServerMaxAllKb() { + return (Long) kp.get(SERVER_MAX_ALL_KB); + } + public void setServerMaxAllKb(long max) { + kp.put(SERVER_MAX_ALL_KB,max); + } + + /** + * Maximum number of fetch successes (e.g. 200 responses) to collect from + * one host. Default is -1, meaning no limit. + */ + { + setHostMaxFetchSuccesses(-1L); // no limit + } + public long getHostMaxFetchSuccesses() { + return (Long) kp.get(HOST_MAX_FETCH_SUCCESSES); + } + public void setHostMaxFetchSuccesses(long max) { + kp.put(HOST_MAX_FETCH_SUCCESSES,max); + } + + /** + * Maximum amount of fetch success content (e.g. 200 responses) in KB to + * collect from one host. Default is -1, meaning no limit. + */ + { + setHostMaxSuccessKb(-1L); // no limit + } + public long getHostMaxSuccessKb() { + return (Long) kp.get(HOST_MAX_SUCCESS_KB); + } + public void setHostMaxSuccessKb(long max) { + kp.put(HOST_MAX_SUCCESS_KB,max); + } + + /** + * Maximum number of fetch responses (incl. error responses) to collect from + * one host. Default is -1, meaning no limit. + */ + { + setHostMaxFetchResponses(-1L); // no limit + } + public long getHostMaxFetchResponses() { + return (Long) kp.get(HOST_MAX_FETCH_RESPONSES); + } + public void setHostMaxFetchResponses(long max) { + kp.put(HOST_MAX_FETCH_RESPONSES,max); + } + + /** + * Maximum amount of response content (incl. error responses) in KB to + * collect from one host. Default is -1, meaning no limit. + */ + { + setHostMaxAllKb(-1L); // no limit + } + public long getHostMaxAllKb() { + return (Long) kp.get(HOST_MAX_ALL_KB); + } + public void setHostMaxAllKb(long max) { + kp.put(HOST_MAX_ALL_KB,max); + } + + /** + * Maximum number of fetch successes (e.g. 200 responses) to collect from + * one group. Default is -1, meaning no limit. + */ + { + setGroupMaxFetchSuccesses(-1L); // no limit + } + public long getGroupMaxFetchSuccesses() { + return (Long) kp.get(GROUP_MAX_FETCH_SUCCESSES); + } + public void setGroupMaxFetchSuccesses(long max) { + kp.put(GROUP_MAX_FETCH_SUCCESSES,max); + } + + /** + * Maximum amount of fetch success content (e.g. 200 responses) in KB to + * collect from one group. Default is -1, meaning no limit. + */ + { + setGroupMaxSuccessKb(-1L); // no limit + } + public long getGroupMaxSuccessKb() { + return (Long) kp.get(GROUP_MAX_SUCCESS_KB); + } + public void setGroupMaxSuccessKb(long max) { + kp.put(GROUP_MAX_SUCCESS_KB,max); + } + + /** + * Maximum number of fetch responses (incl. error responses) to collect from + * one group. Default is -1, meaning no limit. + */ + { + setGroupMaxFetchResponses(-1L); // no limit + } + public long getGroupMaxFetchResponses() { + return (Long) kp.get(GROUP_MAX_FETCH_RESPONSES); + } + public void setGroupMaxFetchResponses(long max) { + kp.put(GROUP_MAX_FETCH_RESPONSES,max); + } + + /** + * Maximum amount of response content (incl. error responses) in KB to + * collect from one group. Default is -1, meaning no limit. + */ + { + setGroupMaxAllKb(-1L); // no limit + } + public long getGroupMaxAllKb() { + return (Long) kp.get(GROUP_MAX_ALL_KB); + } + public void setGroupMaxAllKb(long max) { + kp.put(GROUP_MAX_ALL_KB,max); + } + + /** + * Whether an over-quota situation should result in the containing queue + * being force-retired (if the Frontier supports this). Note that if your + * queues combine URIs that are different with regard to the quota category, + * the retirement may hold back URIs not in the same quota category. Default + * is false. + */ + { + setForceRetire(true); + } + public boolean getForceRetire() { + return (Boolean) kp.get("forceRetire"); + } + public void setForceRetire(boolean force) { + kp.put("forceRetire",force); + } + + protected ServerCache serverCache; + public ServerCache getServerCache() { + return this.serverCache; + } + @Autowired + public void setServerCache(ServerCache serverCache) { + this.serverCache = serverCache; + } + + protected Frontier frontier; + public Frontier getFrontier() { + return this.frontier; + } + @Autowired + public void setFrontier(Frontier frontier) { + this.frontier = frontier; + } + + protected boolean shouldProcess(ProcessorURI puri) { + return puri instanceof CrawlURI; + } + + protected void innerProcess(ProcessorURI puri) { + throw new AssertionError(); + } + + protected ProcessResult innerProcessResult(ProcessorURI puri) { + CrawlURI curi = (CrawlURI)puri; + final CrawlServer server = ServerCacheUtil.getServerFor(serverCache, + curi.getUURI()); + final CrawlHost host = ServerCacheUtil.getHostFor(serverCache, + curi.getUURI()); + FetchStats.HasFetchStats[] haveStats = + new FetchStats.HasFetchStats[] { + server, + host, + frontier.getGroup(curi) + }; + + for(int cat=SERVER;cat<=GROUP;cat++) { + if (checkQuotas(curi,haveStats[cat],cat)) { + return ProcessResult.FINISH; + } + } + + return ProcessResult.PROCEED; + } + + /** + * Check all quotas for the given substats and category (server, host, or + * group). + * + * @param curi CrawlURI to mark up with results + * @param hasStats holds CrawlSubstats with actual values to test + * @param CAT category index (SERVER, HOST, GROUP) to quota settings keys + * @return true if quota precludes fetching of CrawlURI + */ + protected boolean checkQuotas(final CrawlURI curi, + final FetchStats.HasFetchStats hasStats, + final int CAT) { + if (hasStats == null) { + if (LOGGER.isLoggable(Level.FINE)) { + LOGGER.fine(curi.toString() + " null stats category: " + CAT); + } + return false; + } + FetchStats substats = hasStats.getSubstats(); + long[] actuals = new long[] { + substats.getFetchSuccesses(), + substats.getSuccessBytes()/1024, + substats.getFetchResponses(), + substats.getTotalBytes()/1024, + }; + for(int q=SUCCESSES; q<=RESPONSE_KB; q++) { + String key = keys[CAT][q]; + if (applyQuota(curi, key, actuals[q])) { + return true; + } + } + return false; + } + + /** + * Apply the quota specified by the given key against the actual + * value provided. If the quota and actual values rule out processing the + * given CrawlURI, mark up the CrawlURI appropriately. + * + * @param curi CrawlURI whose processing is subject to a potential quota + * limitation + * @param quotaKey settings key to get applicable quota + * @param actual current value to compare to quota + * @return true is CrawlURI is blocked by a quota, false otherwise + */ + protected boolean applyQuota(CrawlURI curi, String key, long actual) { + long quota = (Long)kp.get(key); + if (quota >= 0 && actual >= quota) { + curi.setFetchStatus(S_BLOCKED_BY_QUOTA); + curi.getAnnotations().add("Q:"+key); + if (getForceRetire()) { + curi.setForceRetire(true); + } + return true; + } + return false; + } +} diff --git a/engine/src/main/java/org/archive/crawler/prefetch/RuntimeLimitEnforcer.java b/engine/src/main/java/org/archive/crawler/prefetch/RuntimeLimitEnforcer.java new file mode 100644 index 00000000..e2f7c8ee --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/prefetch/RuntimeLimitEnforcer.java @@ -0,0 +1,200 @@ +/* RuntimeLimitEnforcer + * + * Created on July 7, 2006 + * + * Copyright (C) 2005 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.crawler.prefetch; + + +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.archive.crawler.datamodel.CrawlURI; + +import static org.archive.modules.fetcher.FetchStatusCodes.*; + +import org.archive.crawler.framework.CrawlController; +import org.archive.crawler.framework.CrawlStatus; +import org.archive.crawler.reporting.StatisticsTracker; +import org.archive.modules.ProcessResult; +import org.archive.modules.Processor; +import org.archive.modules.ProcessorURI; +import org.springframework.beans.factory.annotation.Autowired; + + +/** + * A processor to enforce runtime limits on crawls. + *

+ * This processor extends and improves on the 'max-time' capability of Heritrix. + * Essentially, the 'Terminate job' option functions the same way as 'max-time'. + * The processor however also enables pausing when the runtime is exceeded and + * the blocking of all URIs. + *

+ *

    + *
  1. Pause job - Pauses the crawl. A change (increase) to the + * runtime duration will make it pausible to resume the crawl. + * Attempts to resume the crawl without modifying the run time + * will cause it to be immediately paused again.
    2. + *
  2. Terminate job - Terminates the job. Equivalent + * to using the max-time setting on the CrawlController.
    3. + *
  3. Block URIs - Blocks each URI with an -5002 + * (blocked by custom processor) fetch status code. This will + * cause all the URIs queued to wind up in the crawl.log.
    4. + *
      + *

      + * The processor allows variable runtime based on host (or other + * override/refinement criteria) however using such overrides only makes sense + * when using 'Block URIs' as pause and terminate will have global impact once + * encountered anywhere. + * + * @author Kristinn Sigurðsson + */ +public class RuntimeLimitEnforcer extends Processor { + + private static final long serialVersionUID = 3L; + + protected static Logger logger = Logger.getLogger( + RuntimeLimitEnforcer.class.getName()); + + + /** + * The action that the processor takes once the runtime has elapsed. + */ + public static enum Operation { + + /** + * Pauses the crawl. A change (increase) to the runtime duration will + * make it pausible to resume the crawl. Attempts to resume the crawl + * without modifying the run time will cause it to be immediately paused + * again. + */ + PAUSE, + + /** + * Terminates the job. Equivalent to using the max-time setting on the + * CrawlController. + */ + TERMINATE, + + /** + * Blocks each URI with an -5002 (blocked by custom processor) fetch + * status code. This will cause all the URIs queued to wind up in the + * crawl.log. + */ + BLOCK_URIS + }; + + /** + * The amount of time, in seconds, that the crawl will be allowed to run + * before this processor performs it's 'end operation.' + */ + long runtimeSeconds = 24*60*60L; // 1 day + public long getRuntimeSeconds() { + return this.runtimeSeconds; + } + public void setRuntimeSeconds(long secs) { + this.runtimeSeconds = secs; + } + + /** + * The action that the processor takes once the runtime has elapsed. + *

      + * Operation: Pause job - Pauses the crawl. A change (increase) to the + * runtime duration will make it pausible to resume the crawl. Attempts to + * resume the crawl without modifying the run time will cause it to be + * immediately paused again. + *

      + * Operation: Terminate job - Terminates the job. Equivalent to using the + * max-time setting on the CrawlController. + *

      + * Operation: Block URIs - Blocks each URI with an -5002 (blocked by custom + * processor) fetch status code. This will cause all the URIs queued to wind + * up in the crawl.log. + */ + Operation expirationOperation = Operation.PAUSE; + public Operation getExpirationOperation() { + return this.expirationOperation; + } + public void setExpirationOperation(Operation op) { + this.expirationOperation = op; + } + + protected CrawlController controller; + public CrawlController getCrawlController() { + return this.controller; + } + @Autowired + public void setCrawlController(CrawlController controller) { + this.controller = controller; + } + + protected StatisticsTracker statisticsTracker; + public StatisticsTracker getStatisticsTracker() { + return this.statisticsTracker; + } + @Autowired + public void setStatisticsTracker(StatisticsTracker statisticsTracker) { + this.statisticsTracker = statisticsTracker; + } + + public RuntimeLimitEnforcer() { + super(); + } + + + @Override + protected boolean shouldProcess(ProcessorURI puri) { + return puri instanceof CrawlURI; + } + + + @Override + protected void innerProcess(ProcessorURI curi) { + throw new AssertionError(); + } + + @Override + protected ProcessResult innerProcessResult(ProcessorURI curi) + throws InterruptedException { + CrawlController controller = getCrawlController(); + StatisticsTracker stats = getStatisticsTracker(); + long allowedRuntimeMs = getRuntimeSeconds() * 1000L; + long currentRuntimeMs = stats.getCrawlElapsedTime(); + if(currentRuntimeMs > allowedRuntimeMs){ + Operation op = getExpirationOperation(); + if(op != null){ + if (op.equals(Operation.PAUSE)) { + controller.requestCrawlPause(); + } else if (op.equals(Operation.TERMINATE)){ + controller.requestCrawlStop(CrawlStatus.FINISHED_TIME_LIMIT); + } else if (op.equals(Operation.BLOCK_URIS)) { + curi.setFetchStatus(S_BLOCKED_BY_RUNTIME_LIMIT); + curi.getAnnotations().add("Runtime exceeded " + allowedRuntimeMs + + "ms"); + return ProcessResult.FINISH; + } + } else { + logger.log(Level.SEVERE,"Null value for end-operation " + + " when processing " + curi.toString()); + } + } + return ProcessResult.PROCEED; + } +} diff --git a/engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java b/engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java new file mode 100644 index 00000000..46439f19 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java @@ -0,0 +1,347 @@ +/* CrawlMapper + * + * Created on Sep 30, 2005 + * + * Copyright (C) 2005 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.crawler.processor; + +import static org.archive.modules.fetcher.FetchStatusCodes.S_BLOCKED_BY_CUSTOM_PROCESSOR; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.PrintWriter; +import java.util.HashMap; +import java.util.Iterator; + +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.ProcessResult; +import org.archive.modules.Processor; +import org.archive.modules.ProcessorURI; +import org.archive.modules.deciderules.AcceptDecideRule; +import org.archive.modules.deciderules.DecideResult; +import org.archive.modules.deciderules.DecideRule; +import org.archive.spring.ConfigPath; +import org.archive.util.ArchiveUtils; +import org.archive.util.fingerprint.ArrayLongFPCache; +import org.springframework.context.Lifecycle; + +import st.ata.util.FPGenerator; + +/** + * A simple crawl splitter/mapper, dividing up CrawlURIs/CrawlURIs + * between crawlers by diverting some range of URIs to local log files + * (which can then be imported to other crawlers). + * + * May operate on a CrawlURI (typically early in the processing chain) or + * its CrawlURI outlinks (late in the processing chain, after + * LinksScoper), or both (if inserted and configured in both places). + * + *

      Applies a map() method, supplied by a concrete subclass, to + * classKeys to map URIs to crawlers by name. + * + *

      One crawler name is distinguished as the 'local name'; URIs mapped to + * this name are not diverted, but continue to be processed normally. + * + *

      If using the JMX importUris operation importing URLs dropped by + * a {@link CrawlMapper} instance, use recoveryLog style. + * + * @author gojomo + * @version $Date$, $Revision$ + */ +public abstract class CrawlMapper extends Processor implements Lifecycle { + + /** + * PrintWriter which remembers the File to which it writes. + */ + private class FilePrintWriter extends PrintWriter { + File file; + public FilePrintWriter(File file) throws FileNotFoundException { + super(new BufferedOutputStream(new FileOutputStream(file))); + this.file = file; + } + public File getFile() { + return file; + } + } + + + /** + * Whether to apply the mapping to a URI being processed itself, for example + * early in processing (while its status is still 'unattempted'). + */ + boolean checkUri = true; + public boolean getCheckUri() { + return this.checkUri; + } + public void setCheckUri(boolean check) { + this.checkUri = check; + } + + /** + * Whether to apply the mapping to discovered outlinks, for example after + * extraction has occurred. + */ + boolean checkOutlinks = true; + public boolean getCheckOutlinks() { + return this.checkOutlinks; + } + public void setCheckOutlinks(boolean check) { + this.checkOutlinks = check; + } + + /** + * Decide rules to determine if an outlink is subject to mapping. + */ + DecideRule outlinkRule = new AcceptDecideRule(); + public DecideRule getOutlinkRule() { + return this.outlinkRule; + } + public void setOutlinkRule(DecideRule rule) { + this.outlinkRule = rule; + } + + /** + * Name of local crawler node; mappings to this name result in normal + * processing (no diversion). + */ + String localName = "."; + public String getLocalName() { + return this.localName; + } + public void setLocalName(String name) { + this.localName = name; + } + + /** + * Directory to write diversion logs. + */ + ConfigPath diversionDir = + new ConfigPath("diverted URIs subdirectory","diversions"); + public ConfigPath getDiversionDir() { + return this.diversionDir; + } + public void setDiversionDir(ConfigPath path) { + this.diversionDir = path; + } + + /** + * Number of timestamp digits to use as prefix of log names (grouping all + * diversions from that period in a single log). Default is 10 (hourly log + * rotation). + * + */ + int rotationDigits = 10; + public int getRotationDigits() { + return this.rotationDigits; + } + public void setRotationDigits(int digits) { + this.rotationDigits = digits; + } + + /** + * Mapping of target crawlers to logs (PrintWriters) + */ + HashMap diversionLogs + = new HashMap(); + + /** + * Truncated timestamp prefix for diversion logs; when + * current time doesn't match, it's time to close all + * current logs. + */ + String logGeneration = ""; + + protected ArrayLongFPCache cache; + + /** + * Constructor. + * @param name Name of this processor. + */ + public CrawlMapper() { + super(); + } + + + @Override + protected boolean shouldProcess(ProcessorURI puri) { + return true; + } + + @Override + protected void innerProcess(ProcessorURI puri) { + throw new AssertionError(); + } + + @Override + protected ProcessResult innerProcessResult(ProcessorURI puri) { + CrawlURI curi = (CrawlURI)puri; + String nowGeneration = + ArchiveUtils.get14DigitDate().substring( + 0, + getRotationDigits()); + if(!nowGeneration.equals(logGeneration)) { + updateGeneration(nowGeneration); + } + + if (curi.getFetchStatus() <= 0 // unfetched/unsuccessful + && getCheckUri()) { + // apply mapping to the CrawlURI itself + String target = map(curi); + if(!localName.equals(target)) { + // CrawlURI is mapped to somewhere other than here + curi.setFetchStatus(S_BLOCKED_BY_CUSTOM_PROCESSOR); + curi.getAnnotations().add("to:"+target); + divertLog(curi,target); + return ProcessResult.FINISH; + } else { + // localName means keep locally; do nothing + } + } + + if (getCheckOutlinks()) { + // consider outlinks for mapping + Iterator iter = curi.getOutCandidates().iterator(); + while(iter.hasNext()) { + CrawlURI cauri = iter.next(); + if (decideToMapOutlink(cauri)) { + // apply mapping to the CrawlURI + String target = map(cauri); + if(!localName.equals(target)) { + // CrawlURI is mapped to somewhere other than here + iter.remove(); + divertLog(cauri,target); + } else { + // localName means keep locally; do nothing + } + } + } + } + return ProcessResult.PROCEED; + } + + protected boolean decideToMapOutlink(CrawlURI cauri) { + DecideRule rule = getOutlinkRule(); + boolean rejected = rule.decisionFor(cauri) + .equals(DecideResult.REJECT); + return !rejected; + } + + + /** + * Close and mark as finished all existing diversion logs, and + * arrange for new logs to use the new generation prefix. + * + * @param nowGeneration new generation (timestamp prefix) to use + */ + protected synchronized void updateGeneration(String nowGeneration) { + // all existing logs are of a previous generation + Iterator iter = diversionLogs.values().iterator(); + while(iter.hasNext()) { + FilePrintWriter writer = (FilePrintWriter) iter.next(); + writer.close(); + writer.getFile().renameTo( + new File(writer.getFile().getAbsolutePath() + .replaceFirst("\\.open$", ".divert"))); + } + diversionLogs.clear(); + logGeneration = nowGeneration; + } + + /** + * Look up the crawler node name to which the given CrawlURI + * should be mapped. + * + * @param cauri CrawlURI to consider + * @return String node name which should handle URI + */ + protected abstract String map(CrawlURI cauri); + + + /** + * Note the given CrawlURI in the appropriate diversion log. + * + * @param cauri CrawlURI to append to a diversion log + * @param target String node name (log name) to receive URI + */ + protected synchronized void divertLog(CrawlURI cauri, String target) { + if(recentlySeen(cauri)) { + return; + } + PrintWriter diversionLog = getDiversionLog(target); + cauri.singleLineReportTo(diversionLog); + diversionLog.println(); + } + + /** + * Consult the cache to determine if the given URI + * has been recently seen -- entering it if not. + * + * @param cauri CrawlURI to test + * @return true if URI was already in the cache; false otherwise + */ + private boolean recentlySeen(CrawlURI cauri) { + long fp = FPGenerator.std64.fp(cauri.toString()); + return ! cache.add(fp); + } + + /** + * Get the diversion log for a given target crawler node node. + * + * @param target crawler node name of requested log + * @return PrintWriter open on an appropriately-named + * log file + */ + protected PrintWriter getDiversionLog(String target) { + FilePrintWriter writer = (FilePrintWriter) diversionLogs.get(target); + if(writer == null) { + File divertDir = getDiversionDir().getFile(); + divertDir.mkdirs(); + File divertLog = + new File(divertDir, + logGeneration+"-"+localName+"-to-"+target+".open"); + try { + writer = new FilePrintWriter(divertLog); + } catch (FileNotFoundException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + throw new RuntimeException(e); + } + diversionLogs.put(target,writer); + } + return writer; + } + + public void start() { + if(isRunning()) { + return; + } + cache = new ArrayLongFPCache(); + } + + public boolean isRunning() { + return cache != null; + } + + public void stop() { + cache = null; + } +} diff --git a/engine/src/main/java/org/archive/crawler/processor/HashCrawlMapper.java b/engine/src/main/java/org/archive/crawler/processor/HashCrawlMapper.java new file mode 100644 index 00000000..e0050329 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/processor/HashCrawlMapper.java @@ -0,0 +1,139 @@ +/* HashCrawlMapper + * + * Created on Sep 30, 2005 + * + * Copyright (C) 2005 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.crawler.processor; + +import java.util.regex.Matcher; + +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.crawler.framework.Frontier; +import org.archive.net.PublicSuffixes; +import org.archive.util.TextUtils; +import org.springframework.beans.factory.annotation.Autowired; + +import st.ata.util.FPGenerator; + +/** + * Maps URIs to one of N crawler names by applying a hash to the + * URI's (possibly-transformed) classKey. + * + * @author gojomo + * @version $Date$, $Revision$ + */ +public class HashCrawlMapper extends CrawlMapper { + + private static final long serialVersionUID = 2L; + + protected Frontier frontier; + public Frontier getFrontier() { + return this.frontier; + } + @Autowired + public void setFrontier(Frontier frontier) { + this.frontier = frontier; + } + + /** + * Number of crawlers among which to split up the URIs. Their names are + * assumed to be 0..N-1. + */ + long crawlerCount = 1L; + public long getCrawlerCount() { + return this.crawlerCount; + } + public void setCrawlerCount(long count) { + this.crawlerCount = count; + } + + /** + * Whether to use the PublicSuffixes-supplied reduce regex. + * + */ + { + setUsePublicSuffixesRegex(true); + } + public boolean getUsePublicSuffixesRegex() { + return (Boolean) kp.get("usePublicSuffixesRegex"); + } + public void setUsePublicSuffixesRegex(boolean usePublicSuffixes) { + kp.put("usePublicSuffixesRegex",usePublicSuffixes); + } + + /** + * A regex pattern to apply to the classKey, using the first match as the + * mapping key. If empty (the default), use the full classKey. + * + */ + { + setReducePrefixRegex(""); + } + public String getReducePrefixRegex() { + return (String) kp.get("reducePrefixRegex"); + } + public void setReducePrefixRegex(String regex) { + kp.put("reducePrefixRegex",regex); + } + + /** + * Constructor. + */ + public HashCrawlMapper() { + super(); + } + + /** + * Look up the crawler node name to which the given CrawlURI + * should be mapped. + * + * @param cauri CrawlURI to consider + * @return String node name which should handle URI + */ + protected String map(CrawlURI cauri) { + // get classKey, via frontier to generate if necessary + String key = frontier.getClassKey(cauri); + String reduceRegex = getReduceRegex(cauri); + return mapString(key, reduceRegex, getCrawlerCount()); + } + + protected String getReduceRegex(CrawlURI cauri) { + if(getUsePublicSuffixesRegex()) { + return PublicSuffixes.getTopmostAssignedSurtPrefixRegex(); + } else { + return getReducePrefixRegex(); + } + } + + public static String mapString(String key, String reducePattern, + long bucketCount) { + + if (reducePattern != null && reducePattern.length()>0) { + Matcher matcher = TextUtils.getMatcher(reducePattern,key); + if(matcher.find()) { + key = matcher.group(); + } + TextUtils.recycleMatcher(matcher); + } + long fp = FPGenerator.std64.fp(key); + long bucket = fp % bucketCount; + return Long.toString(bucket >= 0 ? bucket : -bucket); + } +} \ No newline at end of file diff --git a/engine/src/main/java/org/archive/crawler/processor/LexicalCrawlMapper.java b/engine/src/main/java/org/archive/crawler/processor/LexicalCrawlMapper.java new file mode 100644 index 00000000..537d617d --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/processor/LexicalCrawlMapper.java @@ -0,0 +1,199 @@ +/* LexicalCrawlMapper + * + * Created on Sep 30, 2005 + * + * Copyright (C) 2005 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.crawler.processor; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.net.URL; +import java.net.URLConnection; +import java.util.Iterator; +import java.util.SortedMap; +import java.util.TreeMap; + +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.crawler.framework.Frontier; +import org.archive.spring.ConfigPath; +import org.archive.util.iterator.LineReadingIterator; +import org.archive.util.iterator.RegexpLineIterator; +import org.springframework.beans.factory.annotation.Autowired; + + +/** + * A simple crawl splitter/mapper, dividing up CrawlURIs/CrawlURIs + * between crawlers by diverting some range of URIs to local log files + * (which can then be imported to other crawlers). + * + * May operate on a CrawlURI (typically early in the processing chain) or + * its CrawlURI outlinks (late in the processing chain, after + * LinksScoper), or both (if inserted and configured in both places). + * + *

      Uses lexical comparisons of classKeys to map URIs to crawlers. The + * 'map' is specified via either a local or HTTP-fetchable file. Each + * line of this file should contain two space-separated tokens, the + * first a key and the second a crawler node name (which should be + * legal as part of a filename). All URIs will be mapped to the crawler + * node name associated with the nearest mapping key equal or subsequent + * to the URI's own classKey. If there are no mapping keys equal or + * after the classKey, the mapping 'wraps around' to the first mapping key. + * + *

      One crawler name is distinguished as the 'local name'; URIs mapped to + * this name are not diverted, but continue to be processed normally. + * + *

      For example, assume a SurtAuthorityQueueAssignmentPolicy and + * a simple mapping file: + * + * 

      + *  d crawlerA
+ *  ~ crawlerB
+ *
      + *

      All URIs with "com," classKeys will find the 'd' key as the nearest + * subsequent mapping key, and thus be mapped to 'crawlerA'. If that's + * the 'local name', the URIs will be processed normally; otherwise, the + * URI will be written to a diversion log aimed for 'crawlerA'. + * + *

      If using the JMX importUris operation importing URLs dropped by + * a {@link LexicalCrawlMapper} instance, use recoveryLog style. + * + * @author gojomo + * @version $Date$, $Revision$ + */ +public class LexicalCrawlMapper extends CrawlMapper { + private static final long serialVersionUID = 2L; + + /** + * Path to map specification file. Each line should include 2 + * whitespace-separated tokens: the first a key indicating the end of a + * range, the second the crawler node to which URIs in the key range should + * be mapped. This setting is ignored if MAP_URI is specified. + */ + ConfigPath mapPath = new ConfigPath("map specification file","lexicalcrawlmapper.config"); + public ConfigPath getMapPath() { + return this.mapPath; + } + public void setMapPath(ConfigPath path) { + this.mapPath = path; + } + + + /** + * URI to map specification file. Each line should include 2 + * whitespace-separated tokens: the first a key indicating the end of a + * range, the second the crawler node to which URIs in the key range should + * be mapped. This setting takes precedence over MAP_PATH; if both are + * specified, then MAP_PATH is ignored. + */ + String mapUri = ""; + public String getMapUri() { + return this.mapUri; + } + public void setMapUri(String uri) { + this.mapUri = uri; + } + + protected Frontier frontier; + public Frontier getFrontier() { + return this.frontier; + } + @Autowired + public void setFrontier(Frontier frontier) { + this.frontier = frontier; + } + + /** + * Mapping of classKey ranges (as represented by their start) to + * crawlers (by abstract name/filename) + */ + TreeMap map = new TreeMap(); + + /** + * Constructor. + */ + public LexicalCrawlMapper() { + super(); + } + + + /** + * Look up the crawler node name to which the given CrawlURI + * should be mapped. + * + * @param cauri CrawlURI to consider + * @return String node name which should handle URI + */ + protected String map(CrawlURI cauri) { + // get classKey, via frontier to generate if necessary + String classKey = frontier.getClassKey(cauri); + SortedMap tail = map.tailMap(classKey); + if(tail.isEmpty()) { + // wraparound + tail = map; + } + // target node is value of nearest subsequent key + return (String) tail.get(tail.firstKey()); + } + + public void start() { + super.start(); + try { + loadMap(); + } catch (IOException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } + + /** + * Retrieve and parse the mapping specification from a local path or + * HTTP URL. + * + * @throws IOException + */ + protected void loadMap() throws IOException { + map.clear(); + String uri = getMapUri(); + Reader reader = null; + if (uri.trim().length() == 0) { + File source = getMapPath().getFile(); + reader = new FileReader(source); + } else { + URLConnection conn = (new URL(uri)).openConnection(); + reader = new InputStreamReader(conn.getInputStream()); + } + reader = new BufferedReader(reader); + Iterator iter = + new RegexpLineIterator( + new LineReadingIterator((BufferedReader) reader), + RegexpLineIterator.COMMENT_LINE, + RegexpLineIterator.TRIMMED_ENTRY_TRAILING_COMMENT, + RegexpLineIterator.ENTRY); + while (iter.hasNext()) { + String[] entry = ((String) iter.next()).split("\\s+"); + map.put(entry[0],entry[1]); + } + reader.close(); + } +} \ No newline at end of file diff --git a/engine/src/main/java/org/archive/crawler/reporting/AlertHandler.java b/engine/src/main/java/org/archive/crawler/reporting/AlertHandler.java new file mode 100644 index 00000000..a70c044d --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/reporting/AlertHandler.java @@ -0,0 +1,78 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.reporting; + +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; + + +/** + * Stub Handler, catching and relaying WARNING/SEVERE events to + * AlertThreadGroup. + * + * @contributor pjack + * @contributor gojomo + */ +public class AlertHandler extends Handler { + // install global AlertHandler + static { + AlertHandler h = new AlertHandler(); + h.setLevel(Level.WARNING); + Logger.getLogger("").addHandler(h); + } + + + @Override + public void close() throws SecurityException { + // Do nothing + } + + + @Override + public void flush() { + // Do nothing + } + + + /** + * Pass record to AlertThreadGroup. + * + * @see java.util.logging.Handler#publish(java.util.logging.LogRecord) + */ + @Override + public void publish(LogRecord record) { + if (!isLoggable(record)) { + return; + } + AlertThreadGroup.publishCurrent(record); + } + + + /** + * Simply to ensure static initialization (installing catchall + * handler on topmost logger) is run. + */ + public static void ensureStaticInitialization() { + // Do nothing + } + +} diff --git a/engine/src/main/java/org/archive/crawler/reporting/AlertThreadGroup.java b/engine/src/main/java/org/archive/crawler/reporting/AlertThreadGroup.java new file mode 100644 index 00000000..5e5fcec6 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/reporting/AlertThreadGroup.java @@ -0,0 +1,110 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.reporting; + +import java.util.LinkedList; +import java.util.logging.LogRecord; +import java.util.logging.Logger; + +import org.archive.io.SinkHandlerLogThread; + +/** + * Parent thread group which lets all child threads find the right + * 'alert' error handler. Helpful for collecting all log events + * of a certain severity (WARNING, SEVERE) from a group of related + * threads. + * + * @contributor pjack + * @contributor gojomo + */ +public class AlertThreadGroup extends ThreadGroup { + protected int count; + protected LinkedList loggers = new LinkedList(); + + public AlertThreadGroup(String name) { + super(name); + } + + public int getAlertCount() { + return count; + } + + public void resetAlertCount() { + count = 0; + } + + public void addLogger(Logger logger) { + loggers.add(logger); + } + + public static AlertThreadGroup current() { + Thread t = Thread.currentThread(); + ThreadGroup th = t.getThreadGroup(); + while ((th != null) && !(th instanceof AlertThreadGroup)) { + th = th.getParent(); + } + return (AlertThreadGroup)th; + } + + public static void publishCurrent(LogRecord record) { + AlertThreadGroup atg = AlertThreadGroup.current(); + if (atg == null) { + return; + } + atg.publish(record); + } + + /** + * Pass a record to all loggers registered with the + * AlertThreadGroup. Adds thread info to the message, + * if available. + * + * @param record + */ + public void publish(LogRecord record) { + String orig = record.getMessage(); + StringBuilder newMessage = new StringBuilder(256); + Thread current = Thread.currentThread(); + newMessage.append(orig).append(" (in thread '"); + newMessage.append(current.getName()).append("'"); + if (current instanceof SinkHandlerLogThread) { + SinkHandlerLogThread tt = (SinkHandlerLogThread)current; + if(tt.getCurrentProcessorName().length()>0) { + newMessage.append("; in processor '"); + newMessage.append(tt.getCurrentProcessorName()); + newMessage.append("'"); + } + } + newMessage.append(")"); + record.setMessage(newMessage.toString()); + count++; + for(Logger logger : loggers) { + // for the relay, suppress use of parent handlers + // (otherwise endless loop a risk if any target + // loggers relay through parents to topmost logger) + synchronized(logger) { + boolean usePar = logger.getUseParentHandlers(); + logger.setUseParentHandlers(false); + logger.log(record); + logger.setUseParentHandlers(usePar); + } + } + } +} diff --git a/engine/src/main/java/org/archive/crawler/reporting/CrawlStatSnapshot.java b/engine/src/main/java/org/archive/crawler/reporting/CrawlStatSnapshot.java new file mode 100644 index 00000000..e6e5cfbf --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/reporting/CrawlStatSnapshot.java @@ -0,0 +1,155 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.crawler.reporting; + +import org.archive.crawler.framework.CrawlController; +import org.archive.util.ArchiveUtils; +import org.archive.util.PaddingStringBuffer; + +/** + * Frozen snapshot of a variety of crawl statistics. Used for + * obtaining a consistent set of stats and a short log of stats + * for calculating rates. + * + * @contributor gojomo + */ +public class CrawlStatSnapshot { + public long timestamp; + + public long urisFetched; + public long bytesProcessed; + + public long discoveredUriCount; + public long queuedUriCount; + public long finishedUriCount; + public long downloadedUriCount; + public long downloadFailures; + public long downloadDisregards; + + public long elapsedMilliseconds; + + public double docsPerSecond; + public double currentDocsPerSecond; + + public long totalKiBPerSec; + public long currentKiBPerSec; + + public int busyThreads; + + public float congestionRatio; + public long deepestUri; + public long averageDepth; + + /** + * Collect all relevant snapshot samples, from the given CrawlController + * and StatisticsTracker (which also provides the previous snapshot + * for rate-calculations. + * + * @param controller + * @param stats + */ + public void collect(CrawlController controller, StatisticsTracker stats) { + // TODO: reconsider names of these methods, inline? + downloadedUriCount = controller.getFrontier().succeededFetchCount(); + bytesProcessed = stats.crawledBytes.getTotal(); + timestamp = System.currentTimeMillis(); + + elapsedMilliseconds = stats.getCrawlElapsedTime(); + discoveredUriCount = controller.getFrontier().discoveredUriCount(); + finishedUriCount = controller.getFrontier().finishedUriCount(); + queuedUriCount = controller.getFrontier().queuedUriCount(); + downloadFailures = controller.getFrontier().failedFetchCount(); + downloadDisregards = controller.getFrontier().disregardedUriCount(); + + busyThreads = controller.getActiveToeCount(); + + congestionRatio = controller.getFrontier().congestionRatio(); + deepestUri = controller.getFrontier().deepestUri(); + averageDepth = controller.getFrontier().averageDepth(); + + // overall rates + docsPerSecond = (double) downloadedUriCount / + (double)(stats.getCrawlElapsedTime() / 1000); + totalKiBPerSec = (long)(((bytesProcessed / 1024) / + (double)((stats.getCrawlElapsedTime()) / 1000)) + .5 ); + + CrawlStatSnapshot lastSnapshot = stats.snapshots.peek(); + + if(lastSnapshot==null) { + // no previous snapshot; unable to calculate current rates + return; + } + + // last sample period rates + long sampleTime = timestamp - lastSnapshot.timestamp; + currentDocsPerSecond = + (double) (downloadedUriCount - lastSnapshot.downloadedUriCount) + / (double)(sampleTime / 1000); + currentKiBPerSec = + (long) (((bytesProcessed-lastSnapshot.bytesProcessed)/1024) + / ((double)sampleTime/1000)); + } + + /** + * Return one line of current progress-statistics + * + * @param now + * @return String of stats + */ + public String getProgressStatisticsLine() { + return new PaddingStringBuffer() + .append(ArchiveUtils.getLog14Date(timestamp)) + .raAppend(32, discoveredUriCount) + .raAppend(44, queuedUriCount) + .raAppend(57, downloadedUriCount) + .raAppend(74, ArchiveUtils. + doubleToString(currentDocsPerSecond, 2) + + "(" + ArchiveUtils.doubleToString(docsPerSecond, 2) + ")") + .raAppend(85, currentKiBPerSec + "(" + totalKiBPerSec + ")") + .raAppend(99, downloadFailures) + .raAppend(113, busyThreads) + .raAppend(126, (Runtime.getRuntime().totalMemory() - + Runtime.getRuntime().freeMemory()) / 1024) + .raAppend(140, Runtime.getRuntime().totalMemory() / 1024) + .raAppend(153, ArchiveUtils.doubleToString(congestionRatio, 2)) + .raAppend(165, deepestUri) + .raAppend(177, averageDepth) + .toString(); + } + + public long totalCount() { + return queuedUriCount + busyThreads + + downloadedUriCount; + } + + /** + * This returns the number of completed URIs as a percentage of the total + * number of URIs encountered (should be inverse to the discovery curve) + * + * @return The number of completed URIs as a percentage of the total + * number of URIs encountered + */ + public int percentOfDiscoveredUrisCompleted() { + long total = discoveredUriCount; + if (total == 0) { + return 0; + } + return (int) (100 * finishedUriCount / total); + } +} diff --git a/engine/src/main/java/org/archive/crawler/reporting/CrawlSummaryReport.java b/engine/src/main/java/org/archive/crawler/reporting/CrawlSummaryReport.java new file mode 100644 index 00000000..00501b9b --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/reporting/CrawlSummaryReport.java @@ -0,0 +1,82 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.reporting; + +import java.io.PrintWriter; + +import org.archive.crawler.util.CrawledBytesHistotable; +import org.archive.util.ArchiveUtils; + +/** + * The "Crawl Report", with summaries of overall crawl size. + * + * @contributor gojomo + */ +public class CrawlSummaryReport extends Report { + + @Override + public void write(PrintWriter writer) { + CrawlStatSnapshot snapshot = stats.getLastSnapshot(); + writer.print("Crawl Name: " + stats.getCrawlController().getMetadata().getJobName()); + writer.print("\nCrawl Status: " + stats.getCrawlController().getCrawlExitStatus().desc); + writer.print("\nDuration Time: " + + ArchiveUtils.formatMillisecondsToConventional(stats.getCrawlElapsedTime())); + stats.tallySeeds(); + writer.print("\nTotal Seeds Crawled: " + stats.seedsCrawled); + writer.print("\nTotal Seeds not Crawled: " + (stats.seedsTotal - stats.seedsCrawled)); + // hostsDistribution contains all hosts crawled plus an entry for dns. + writer.print("\nTotal Hosts Crawled: " + (stats.hostsDistribution.size()-1)); + writer.print("\nTotal URIs Processed: " + snapshot.finishedUriCount); + writer.print("\nURIs Crawled successfully: " + snapshot.downloadedUriCount); + writer.print("\nURIs Failed to Crawl: " + snapshot.downloadFailures); + writer.print("\nURIs Disregarded: " + snapshot.downloadDisregards); + writer.print("\nProcessed docs/sec: " + + ArchiveUtils.doubleToString(snapshot.docsPerSecond,2)); + writer.print("\nBandwidth in Kbytes/sec: " + snapshot.totalKiBPerSec); + writer.print("\nTotal Raw Data Size in Bytes: " + snapshot.bytesProcessed + + " (" + ArchiveUtils.formatBytesForDisplay(snapshot.bytesProcessed) + + ") \n"); + writer.print("Novel Bytes: " + + stats.crawledBytes.get(CrawledBytesHistotable.NOVEL) + + " (" + ArchiveUtils.formatBytesForDisplay( + stats.crawledBytes.get(CrawledBytesHistotable.NOVEL)) + + ") \n"); + if(stats.crawledBytes.containsKey(CrawledBytesHistotable.DUPLICATE)) { + writer.print("Duplicate-by-hash Bytes: " + + stats.crawledBytes.get(CrawledBytesHistotable.DUPLICATE) + + " (" + ArchiveUtils.formatBytesForDisplay( + stats.crawledBytes.get(CrawledBytesHistotable.DUPLICATE)) + + ") \n"); + } + if(stats.crawledBytes.containsKey(CrawledBytesHistotable.NOTMODIFIED)) { + writer.print("Not-modified Bytes: " + + stats.crawledBytes.get(CrawledBytesHistotable.NOTMODIFIED) + + " (" + ArchiveUtils.formatBytesForDisplay( + stats.crawledBytes.get(CrawledBytesHistotable.NOTMODIFIED)) + + ") \n"); + } + } + + @Override + public String getFilename() { + return "crawl-report.txt"; + } + +} diff --git a/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java b/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java new file mode 100644 index 00000000..82ac93b2 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java @@ -0,0 +1,442 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.reporting; + +import java.io.File; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.FileHandler; +import java.util.logging.Formatter; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.logging.SimpleFormatter; + +import org.apache.commons.httpclient.URIException; +import org.archive.checkpointing.Checkpointable; +import org.archive.checkpointing.RecoverAction; +import org.archive.crawler.framework.Engine; +import org.archive.crawler.io.NonFatalErrorFormatter; +import org.archive.crawler.io.RuntimeErrorFormatter; +import org.archive.crawler.io.StatisticsLogFormatter; +import org.archive.crawler.io.UriErrorFormatter; +import org.archive.crawler.io.UriProcessingFormatter; +import org.archive.crawler.util.Logs; +import org.archive.io.GenerationFileHandler; +import org.archive.modules.extractor.UriErrorLoggerModule; +import org.archive.net.UURI; +import org.archive.net.UURIFactory; +import org.archive.spring.ConfigPath; +import org.archive.util.ArchiveUtils; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.context.Lifecycle; + +/** + * Module providing all expected whole-crawl logging facilities + * + * @contributor pjack + * @contributor gojomo + */ +public class CrawlerLoggerModule + implements + UriErrorLoggerModule, Lifecycle, InitializingBean, + Checkpointable { + private static final long serialVersionUID = 1L; + + protected ConfigPath path = new ConfigPath(Engine.LOGS_DIR_NAME,"logs"); + public ConfigPath getPath() { + return path; + } + public void setPath(ConfigPath path) { + this.path = path.merge(this.path); + } + + // manifest support + /** abbrieviation label for config files in manifest */ + public static final char MANIFEST_CONFIG_FILE = 'C'; + /** abbrieviation label for report files in manifest */ + public static final char MANIFEST_REPORT_FILE = 'R'; + /** abbrieviation label for log files in manifest */ + public static final char MANIFEST_LOG_FILE = 'L'; + + + // key log names + private static final String LOGNAME_CRAWL = "crawl"; + private static final String LOGNAME_ALERTS = "alerts"; + private static final String LOGNAME_PROGRESS_STATISTICS = + "progress-statistics"; + private static final String LOGNAME_URI_ERRORS = "uri-errors"; + private static final String LOGNAME_RUNTIME_ERRORS = "runtime-errors"; + private static final String LOGNAME_NONFATAL_ERRORS = "nonfatal-errors"; + + + protected ConfigPath crawlLogPath = + new ConfigPath(Logs.CRAWL.getFilename(),Logs.CRAWL.getFilename()); + public ConfigPath getCrawlLogPath() { + return crawlLogPath; + } + public void setCrawlLogPath(ConfigPath cp) { + this.crawlLogPath = cp.merge(this.crawlLogPath); + } + + protected ConfigPath alertsLogPath = + new ConfigPath(Logs.ALERTS.getFilename(),Logs.ALERTS.getFilename()); + public ConfigPath getAlertsLogPath() { + return alertsLogPath; + } + public void setAlertsLogPath(ConfigPath cp) { + this.alertsLogPath = cp.merge(this.alertsLogPath); + } + + protected ConfigPath progressLogPath = + new ConfigPath(Logs.PROGRESS_STATISTICS.getFilename(),Logs.PROGRESS_STATISTICS.getFilename()); + public ConfigPath getProgressLogPath() { + return progressLogPath; + } + public void setProgressLogPath(ConfigPath cp) { + this.progressLogPath = cp.merge(this.progressLogPath); + } + + protected ConfigPath uriErrorsLogPath = + new ConfigPath(Logs.URI_ERRORS.getFilename(),Logs.URI_ERRORS.getFilename()); + public ConfigPath getUriErrorsLogPath() { + return uriErrorsLogPath; + } + public void setUriErrorsLogPath(ConfigPath cp) { + this.uriErrorsLogPath = cp.merge(this.uriErrorsLogPath); + } + + protected ConfigPath runtimeErrorsLogPath = + new ConfigPath(Logs.RUNTIME_ERRORS.getFilename(),Logs.RUNTIME_ERRORS.getFilename()); + public ConfigPath getRuntimeErrorsLogPath() { + return runtimeErrorsLogPath; + } + public void setRuntimeErrorsLogPath(ConfigPath cp) { + this.runtimeErrorsLogPath = cp.merge(this.runtimeErrorsLogPath); + } + + protected ConfigPath nonfatalErrorsLogPath = + new ConfigPath(Logs.NONFATAL_ERRORS.getFilename(),Logs.NONFATAL_ERRORS.getFilename()); + public ConfigPath getNonfatalErrorsLogPath() { + return nonfatalErrorsLogPath; + } + public void setNonfatalErrorsLogPath(ConfigPath cp) { + this.nonfatalErrorsLogPath = cp.merge(this.nonfatalErrorsLogPath); + } + + /** suffix to use on active logs */ +// public static final String CURRENT_LOG_SUFFIX = ".log"; + + /** + * Crawl progress logger. + * + * No exceptions. Logs summary result of each url processing. + */ + private transient Logger uriProcessing; + + /** + * This logger contains unexpected runtime errors. + * + * Would contain errors trying to set up a job or failures inside + * processors that they are not prepared to recover from. + */ + private transient Logger runtimeErrors; + + /** + * This logger is for job-scoped logging, specifically recoverable + * errors which happen and are handled within a particular processor. + * + * Examples would be socket timeouts, exceptions thrown by + * extractors, etc. + */ + private transient Logger nonfatalErrors; + + /** + * Special log for URI format problems, wherever they may occur. + */ + private transient Logger uriErrors; + + /** + * Statistics tracker writes here at regular intervals. + */ + private transient Logger progressStats; + + /** + * Logger to hold job summary report. + * + * Large state reports made at infrequent intervals (e.g. job ending) go + * here. + */ + private transient Logger reports; + + /** + * Record of fileHandlers established for loggers, + * assisting file rotation. + */ + transient private Map fileHandlers; + + private StringBuffer manifest = new StringBuffer(); + + private transient AlertThreadGroup atg; + + public CrawlerLoggerModule() { + + } + + public void start() { + if(isRunning) { + return; + } + getPath().getFile().mkdirs(); + this.atg = AlertThreadGroup.current(); + try { + setupLogs(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + isRunning = true; + } + + boolean isRunning = false; + public boolean isRunning() { + return this.isRunning; + } + + public void stop() { + closeLogFiles(); + isRunning = false; + } + + private void setupLogs() throws IOException { + String logsPath = getPath().getFile().getAbsolutePath() + File.separatorChar; + uriProcessing = Logger.getLogger(LOGNAME_CRAWL + "." + logsPath); + runtimeErrors = Logger.getLogger(LOGNAME_RUNTIME_ERRORS + "." + + logsPath); + nonfatalErrors = Logger.getLogger(LOGNAME_NONFATAL_ERRORS + "." + logsPath); + uriErrors = Logger.getLogger(LOGNAME_URI_ERRORS + "." + logsPath); + progressStats = Logger.getLogger(LOGNAME_PROGRESS_STATISTICS + "." + + logsPath); + + this.fileHandlers = new HashMap(); + setupLogFile(uriProcessing, + getCrawlLogPath().getFile().getAbsolutePath(), + new UriProcessingFormatter(), true); + + setupLogFile(runtimeErrors, + getRuntimeErrorsLogPath().getFile().getAbsolutePath(), + new RuntimeErrorFormatter(), true); + + setupLogFile(nonfatalErrors, + getNonfatalErrorsLogPath().getFile().getAbsolutePath(), + new NonFatalErrorFormatter(), true); + + setupLogFile(uriErrors, + getUriErrorsLogPath().getFile().getAbsolutePath(), + new UriErrorFormatter(), true); + + setupLogFile(progressStats, + getProgressLogPath().getFile().getAbsolutePath(), + new StatisticsLogFormatter(), true); + + setupAlertLog(logsPath); + } + + private void setupLogFile(Logger logger, String filename, Formatter f, + boolean shouldManifest) throws IOException, SecurityException { + logger.setLevel(Level.INFO); // set all standard loggers to INFO + GenerationFileHandler fh = GenerationFileHandler.makeNew(filename, false, + shouldManifest); + fh.setFormatter(f); + logger.addHandler(fh); + addToManifest(filename, MANIFEST_LOG_FILE, shouldManifest); + logger.setUseParentHandlers(false); + this.fileHandlers.put(logger, fh); + } + + + private void setupAlertLog(String logsPath) throws IOException { + Logger logger = Logger.getLogger(LOGNAME_ALERTS + "." + logsPath); + String filename = getAlertsLogPath().getFile().getAbsolutePath(); + GenerationFileHandler fh = + GenerationFileHandler.makeNew(filename, false, true); + fh.setFormatter(new SimpleFormatter()); + AlertThreadGroup.current().addLogger(logger); + AlertHandler.ensureStaticInitialization(); + logger.addHandler(fh); + addToManifest(filename, MANIFEST_LOG_FILE, true); + logger.setUseParentHandlers(false); + this.fileHandlers.put(logger, fh); + } + + + public void rotateLogFiles() throws IOException { + rotateLogFiles("." + ArchiveUtils.get14DigitDate()); + } + + protected void rotateLogFiles(String generationSuffix) + throws IOException { + for (Logger l: fileHandlers.keySet()) { + GenerationFileHandler gfh = + (GenerationFileHandler)fileHandlers.get(l); + GenerationFileHandler newGfh = + gfh.rotate(generationSuffix, ""); + if (gfh.shouldManifest()) { + addToManifest((String) newGfh.getFilenameSeries().get(1), + MANIFEST_LOG_FILE, newGfh.shouldManifest()); + } + l.removeHandler(gfh); + l.addHandler(newGfh); + fileHandlers.put(l, newGfh); + } + } + + /** + * Close all log files and remove handlers from loggers. + */ + public void closeLogFiles() { + for (Logger l: fileHandlers.keySet()) { + GenerationFileHandler gfh = + (GenerationFileHandler)fileHandlers.get(l); + gfh.close(); + l.removeHandler(gfh); + } + } + + + /** + * Add a file to the manifest of files used/generated by the current + * crawl. + * + * TODO: Its possible for a file to be added twice if reports are + * force generated midcrawl. Fix. + * + * @param file The filename (with absolute path) of the file to add + * @param type The type of the file + * @param bundle Should the file be included in a typical bundling of + * crawler files. + * + * @see #MANIFEST_CONFIG_FILE + * @see #MANIFEST_LOG_FILE + * @see #MANIFEST_REPORT_FILE + */ + public void addToManifest(String file, char type, boolean bundle) { + manifest.append(type + (bundle? "+": "-") + " " + file + "\n"); + } + + + /** + * Run checkpointing. + * + *

      Default access only to be called by Checkpointer. + * @throws Exception + */ + public void checkpoint(File checkpointDir, List actions) + throws IOException { + // Rotate off crawler logs. + rotateLogFiles("." + checkpointDir.getName()); +// this.checkpointer.getNextCheckpointName()); + } + + public Logger getNonfatalErrors() { + return nonfatalErrors; + } + + + public Logger getProgressStats() { + return progressStats; + } + + + public Logger getReports() { + return reports; + } + + + public Logger getRuntimeErrors() { + return runtimeErrors; + } + + + public Logger getUriErrors() { + return uriErrors; + } + + + public Logger getUriProcessing() { + return uriProcessing; + } + + + public int getAlertCount() { + if (atg != null) { + return atg.getAlertCount(); + } else { + return -1; + } + } + + + public void resetAlertCount() { + if (atg != null) { + atg.resetAlertCount(); + } + } + + + /** + * Log a URIException from deep inside other components to the crawl's + * shared log. + * + * @param e URIException encountered + * @param u CrawlURI where problem occurred + * @param l String which could not be interpreted as URI without exception + */ + public void logUriError(URIException e, UURI u, CharSequence l) { + if (e.getReasonCode() == UURIFactory.IGNORED_SCHEME) { + // don't log those that are intentionally ignored + return; + } + Object[] array = {u, l}; + uriErrors.log(Level.INFO, e.getMessage(), array); + } + + + private void readObject(ObjectInputStream in) + throws IOException, ClassNotFoundException { + in.defaultReadObject(); + getPath().getFile().mkdirs(); + this.atg = AlertThreadGroup.current(); + this.setupLogs(); + } + + + public void afterPropertiesSet() throws Exception { + ConfigPath[] paths = { + crawlLogPath, alertsLogPath, progressLogPath, + uriErrorsLogPath, runtimeErrorsLogPath, nonfatalErrorsLogPath }; + for(ConfigPath cp : paths) { + if(cp.getBase()==null) { + cp.setBase(getPath()); + } + } + } +} diff --git a/engine/src/main/java/org/archive/crawler/reporting/FrontierReport.java b/engine/src/main/java/org/archive/crawler/reporting/FrontierReport.java new file mode 100644 index 00000000..309beca0 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/reporting/FrontierReport.java @@ -0,0 +1,21 @@ +package org.archive.crawler.reporting; + +import java.io.PrintWriter; + +public class FrontierReport extends Report { + + @Override + public void write(PrintWriter writer) { + if(stats.controller.getFrontier().isEmpty()) { + writer.println("frontier empty"); + } else { + stats.controller.getFrontier().reportTo("nonempty", writer); + } + } + + @Override + public String getFilename() { + return "frontier-report.txt"; + } + +} diff --git a/engine/src/main/java/org/archive/crawler/reporting/HostsReport.java b/engine/src/main/java/org/archive/crawler/reporting/HostsReport.java new file mode 100644 index 00000000..9cd1f70b --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/reporting/HostsReport.java @@ -0,0 +1,85 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.reporting; + +import java.io.PrintWriter; +import java.util.Iterator; +import java.util.SortedMap; + +import org.apache.commons.collections.Closure; +import org.archive.modules.net.CrawlHost; +import org.archive.util.LongWrapper; + +/** + * The "Hosts Report", tallies by host. + * + * @contributor gojomo + */ +public class HostsReport extends Report { + + @Override + public void write(final PrintWriter writer) { + // TODO: use CrawlHosts for all stats; only perform sorting on + // manageable number of hosts + SortedMap hd = stats.getReverseSortedHostsDistribution(); + // header + writer.print("[#urls] [#bytes] [host] [#robots] [#remaining]\n"); + for (Iterator i = hd.keySet().iterator(); i.hasNext();) { + // Key is 'host'. + String key = (String) i.next(); + CrawlHost host = stats.serverCache.getHostFor(key); + LongWrapper val = (LongWrapper)hd.get(key); + writeReportLine(writer, + ((val==null)?"-":val.longValue), + stats.getBytesPerHost(key), + key, + host.getSubstats().getRobotsDenials(), + host.getSubstats().getRemaining()); + } + // StatisticsTracker doesn't know of zero-completion hosts; + // so supplement report with those entries from host cache + Closure logZeros = new Closure() { + public void execute(Object obj) { + CrawlHost host = (CrawlHost)obj; + if(host.getSubstats().getRecordedFinishes()==0) { + writeReportLine(writer, + host.getSubstats().getRecordedFinishes(), + host.getSubstats().getTotalBytes(), + host.getHostName(), + host.getSubstats().getRobotsDenials(), + host.getSubstats().getRemaining()); + } + }}; + stats.serverCache.forAllHostsDo(logZeros); + } + + protected void writeReportLine(PrintWriter writer, Object ... fields) { + for(Object field : fields) { + writer.print(field); + writer.print(" "); + } + writer.print("\n"); + } + + @Override + public String getFilename() { + return "hosts-report.txt"; + } +} diff --git a/engine/src/main/java/org/archive/crawler/reporting/MimetypesReport.java b/engine/src/main/java/org/archive/crawler/reporting/MimetypesReport.java new file mode 100644 index 00000000..80068a00 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/reporting/MimetypesReport.java @@ -0,0 +1,56 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.reporting; + +import java.io.PrintWriter; +import java.util.Iterator; +import java.util.TreeMap; + +import org.archive.util.LongWrapper; + +/** + * The "Mimetypes Report", tallies by MIME type. + * + * @contributor gojomo + */ +public class MimetypesReport extends Report { + + @Override + public void write(PrintWriter writer) { + // header + writer.print("[#urls] [#bytes] [mime-types]\n"); + TreeMap fd = stats.getReverseSortedCopy(stats.getFileDistribution()); + for (Iterator i = fd.keySet().iterator(); i.hasNext();) { + Object key = i.next(); + // Key is mime type. + writer.print(Long.toString(((LongWrapper)fd.get(key)).longValue)); + writer.print(" "); + writer.print(Long.toString(stats.getBytesPerFileType((String)key))); + writer.print(" "); + writer.print((String)key); + writer.print("\n"); + } } + + @Override + public String getFilename() { + return "mimetype-report.txt"; + } + +} diff --git a/engine/src/main/java/org/archive/crawler/reporting/ProcessorsReport.java b/engine/src/main/java/org/archive/crawler/reporting/ProcessorsReport.java new file mode 100644 index 00000000..a29b958f --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/reporting/ProcessorsReport.java @@ -0,0 +1,42 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.crawler.reporting; + +import java.io.PrintWriter; + +/** + * The "Processors Report", delegated through the CrawlController + * to each Processor to dump whatever information it collects for + * this purpose. + * + * @contributor gojomo + */ +public class ProcessorsReport extends Report { + + @Override + public void write(PrintWriter writer) { + stats.controller.getProcessorChain().reportTo(writer); + } + + @Override + public String getFilename() { + return "processors-report.txt"; + } + +} diff --git a/engine/src/main/java/org/archive/crawler/reporting/Report.java b/engine/src/main/java/org/archive/crawler/reporting/Report.java new file mode 100644 index 00000000..7fe0fdf0 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/reporting/Report.java @@ -0,0 +1,48 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.crawler.reporting; + +import java.io.PrintWriter; + +/** + * Abstract superclass for named crawl reports that need only a + * StatisticsTracker and can dump a plain-text representation to a + * PrintWriter. + * + * @contributor gojomo + */ +public abstract class Report { + StatisticsTracker stats; + + public Report() { + } + + public abstract void write(PrintWriter writer); + + public abstract String getFilename(); + + public StatisticsTracker getStats() { + return stats; + } + + public Report setStats(StatisticsTracker stats) { + this.stats = stats; + return this; + } +} diff --git a/engine/src/main/java/org/archive/crawler/reporting/ResponseCodeReport.java b/engine/src/main/java/org/archive/crawler/reporting/ResponseCodeReport.java new file mode 100644 index 00000000..fbd15322 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/reporting/ResponseCodeReport.java @@ -0,0 +1,55 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.crawler.reporting; + +import java.io.PrintWriter; +import java.util.Iterator; +import java.util.TreeMap; + +import org.archive.util.LongWrapper; + +/** + * The "Response Codes Report", tallies by response/disposition code. + * + * @contributor gojomo + */ +public class ResponseCodeReport extends Report { + + @Override + public void write(PrintWriter writer) { + // header + writer.print("[rescode] [#urls]\n"); + + TreeMap scd = + stats.getReverseSortedCopy(stats.getStatusCodeDistribution()); + for (Iterator i = scd.keySet().iterator(); i.hasNext();) { + Object key = i.next(); + writer.print((String)key); + writer.print(" "); + writer.print(Long.toString(((LongWrapper)scd.get(key)).longValue)); + writer.print("\n"); + } + } + + @Override + public String getFilename() { + return "responsecode-report.txt"; + } + +} diff --git a/engine/src/main/java/org/archive/crawler/reporting/SeedRecord.java b/engine/src/main/java/org/archive/crawler/reporting/SeedRecord.java new file mode 100644 index 00000000..50ab0c3d --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/reporting/SeedRecord.java @@ -0,0 +1,116 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.reporting; + +import java.io.Serializable; + +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.crawler.datamodel.CoreAttributeConstants; + + +/** + * Record of all interesting info about the most-recent + * processing of a specific seed. + * + * @author gojomo + */ +public class SeedRecord implements CoreAttributeConstants, Serializable { + private static final long serialVersionUID = -8455358640509744478L; + private final String uri; + private int statusCode; + private final String disposition; + private String redirectUri; + + /** + * Create a record from the given CrawlURI and disposition string + * + * @param curi CrawlURI, already processed as reported to StatisticsTracker + * @param disposition descriptive disposition string + * + */ + public SeedRecord(CrawlURI curi, String disposition) { + super(); + this.uri = curi.toString(); + this.statusCode = curi.getFetchStatus(); + this.disposition = disposition; + if (statusCode==301 || statusCode == 302) { + for (CrawlURI cauri: curi.getOutCandidates()) { + if("location:".equalsIgnoreCase(cauri.getViaContext(). + toString())) { + redirectUri = cauri.toString(); + } + } + } + } + + /** + * Constructor for when a CrawlURI is unavailable; such + * as when considering seeds not yet passed through as + * CrawlURIs. + * + * @param uri + * @param disposition + */ + public SeedRecord(String uri, String disposition) { + this(uri, disposition, -1, null); + } + + /** + * Create a record from the given URI, disposition, HTTP status code, + * and redirect URI. + * @param uri + * @param disposition + * @param statusCode + * @param redirectUri + */ + public SeedRecord(String uri, String disposition, int statusCode, + String redirectUri) { + super(); + this.uri = uri; + this.statusCode = statusCode; + this.disposition = disposition; + this.redirectUri = redirectUri; + } + + /** + * @return Returns the disposition. + */ + public String getDisposition() { + return disposition; + } + /** + * @return Returns the redirectUri. + */ + public String getRedirectUri() { + return redirectUri; + } + /** + * @return Returns the statusCode. + */ + public int getStatusCode() { + return statusCode; + } + /** + * @return Returns the uri. + */ + public String getUri() { + return uri; + } +} \ No newline at end of file diff --git a/engine/src/main/java/org/archive/crawler/reporting/SeedsReport.java b/engine/src/main/java/org/archive/crawler/reporting/SeedsReport.java new file mode 100644 index 00000000..bb81ede5 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/reporting/SeedsReport.java @@ -0,0 +1,68 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.crawler.reporting; + +import java.io.PrintWriter; +import java.util.Iterator; + + +/** + * The "Seeds Report", results per provided seed. + * + * @contributor gojomo + */ +public class SeedsReport extends Report { + + @Override + public void write(PrintWriter writer) { + // Build header. + writer.print("[code] [status] [seed] [redirect]\n"); + + long seedsCrawled = 0; + long seedsTotal = 0; + for (Iterator i = stats.getSeedRecordsSortedByStatusCode(stats.getSeedsIterator()); + i.hasNext();) { + SeedRecord sr = (SeedRecord)i.next(); + writer.print(sr.getStatusCode()); + writer.print(" "); + seedsTotal++; + if((sr.getStatusCode() > 0)) { + seedsCrawled++; + writer.print("CRAWLED"); + } else { + writer.print("NOTCRAWLED"); + } + writer.print(" "); + writer.print(sr.getUri()); + if(sr.getRedirectUri()!=null) { + writer.print(" "); + writer.print(sr.getRedirectUri()); + } + writer.print("\n"); + } + stats.seedsTotal = seedsTotal; + stats.seedsCrawled = seedsCrawled; + } + + @Override + public String getFilename() { + return "seeds-report.txt"; + } + +} diff --git a/engine/src/main/java/org/archive/crawler/reporting/SourceTagsReport.java b/engine/src/main/java/org/archive/crawler/reporting/SourceTagsReport.java new file mode 100644 index 00000000..01d7c4b0 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/reporting/SourceTagsReport.java @@ -0,0 +1,65 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.crawler.reporting; + +import java.io.PrintWriter; +import java.util.Iterator; +import java.util.Map; +import java.util.SortedMap; + +import org.archive.util.LongWrapper; + +/** + * The "Source Report", tallies of source tags (usually seeds) by host. + * + * @contributor gojomo + */ +public class SourceTagsReport extends Report { + + @Override + public void write(PrintWriter writer) { + + writer.print("[source] [host] [#urls]\n"); + // for each source + for (Iterator i = stats.sourceHostDistribution.keySet().iterator(); i.hasNext();) { + String sourceKey = i.next(); + Map hostCounts = + (Map)stats.sourceHostDistribution.get(sourceKey); + // sort hosts by #urls + SortedMap sortedHostCounts = + stats.getReverseSortedHostCounts(hostCounts); + // for each host + for (Iterator j = sortedHostCounts.keySet().iterator(); j.hasNext();) { + Object hostKey = j.next(); + LongWrapper hostCount = (LongWrapper) hostCounts.get(hostKey); + writer.print(sourceKey.toString()); + writer.print(" "); + writer.print(hostKey.toString()); + writer.print(" "); + writer.print(hostCount.longValue); + writer.print("\n"); + } + } + } + + @Override + public String getFilename() { + return "source-report.txt"; + } +} diff --git a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java new file mode 100644 index 00000000..044fbd80 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java @@ -0,0 +1,942 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.reporting; + +import static org.archive.crawler.datamodel.CoreAttributeConstants.A_SOURCE_TAG; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.Serializable; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.Hashtable; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.Vector; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.archive.bdb.BdbModule; +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.crawler.event.CrawlStateEvent; +import org.archive.crawler.event.CrawlURIDispositionEvent; +import org.archive.crawler.event.StatSnapshotEvent; +import org.archive.crawler.framework.CrawlController; +import org.archive.crawler.framework.Engine; +import org.archive.crawler.util.CrawledBytesHistotable; +import org.archive.crawler.util.TopNSet; +import org.archive.modules.net.ServerCache; +import org.archive.modules.net.ServerCacheUtil; +import org.archive.modules.seeds.SeedModule; +import org.archive.net.UURI; +import org.archive.spring.ConfigPath; +import org.archive.util.ArchiveUtils; +import org.archive.util.LongWrapper; +import org.archive.util.MimetypeUtils; +import org.archive.util.PaddingStringBuffer; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.context.ApplicationEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.context.Lifecycle; +import org.xbill.DNS.DClass; +import org.xbill.DNS.Lookup; + +import com.sleepycat.je.DatabaseException; + +/** + * This is an implementation of the AbstractTracker. It is designed to function + * with the WUI as well as performing various logging activity. + *

      + * At the end of each snapshot a line is written to the + * 'progress-statistics.log' file. + *

      + * The header of that file is as follows: + * 

       [timestamp] [discovered]    [queued] [downloaded] [doc/s(avg)]  [KB/s(avg)] [dl-failures] [busy-thread] [mem-use-KB]
      + * First there is a timestamp, accurate down to 1 second. + *

      + * discovered, queued, downloaded and dl-failures + * are (respectively) the discovered URI count, pending URI count, successfully + * fetched count and failed fetch count from the frontier at the time of the + * snapshot. + *

      + * KB/s(avg) is the bandwidth usage. We use the total bytes downloaded + * to calculate average bandwidth usage (KB/sec). Since we also note the value + * each time a snapshot is made we can calculate the average bandwidth usage + * during the last snapshot period to gain a "current" rate. The first number is + * the current and the average is in parenthesis. + *

      + * doc/s(avg) works the same way as doc/s except it show the number of + * documents (URIs) rather then KB downloaded. + *

      + * busy-threads is the total number of ToeThreads that are not available + * (and thus presumably busy processing a URI). This information is extracted + * from the crawl controller. + *

      + * Finally mem-use-KB is extracted from the run time environment + * (Runtime.getRuntime().totalMemory()). + *

      + * In addition to the data collected for the above logs, various other data + * is gathered and stored by this tracker. + *

        + *
      • Successfully downloaded documents per fetch status code + *
      • Successfully downloaded documents per document mime type + *
      • Amount of data per mime type + *
      • Successfully downloaded documents per host + *
      • Amount of data per host + *
      • Disposition of all seeds (this is written to 'reports.log' at end of + * crawl) + *
      • Successfully downloaded documents per host per source + *
      + * + * @contributor Parker Thompson + * @contributor Kristinn Sigurdsson + * @contributor gojomo + */ +public class StatisticsTracker + implements + ApplicationContextAware, + ApplicationListener, + Lifecycle, + Runnable, + Serializable { + private static final long serialVersionUID = 5L; + + protected SeedModule seeds; + public SeedModule getSeeds() { + return this.seeds; + } + @Autowired + public void setSeeds(SeedModule seeds) { + this.seeds = seeds; + } + + protected BdbModule bdb; + @Autowired + public void setBdbModule(BdbModule bdb) { + this.bdb = bdb; + } + + protected ConfigPath reportsDir = new ConfigPath(Engine.REPORTS_DIR_NAME,"."); + public ConfigPath getReportsDir() { + return reportsDir; + } + public void setReportsDir(ConfigPath reportsDir) { + this.reportsDir = reportsDir; + } + + protected ServerCache serverCache; + public ServerCache getServerCache() { + return this.serverCache; + } + @Autowired + public void setServerCache(ServerCache serverCache) { + this.serverCache = serverCache; + } + + protected int liveHostReportSize = 20; + public int getLiveHostReportSize() { + return liveHostReportSize; + } + public void setLiveHostReportSize(int liveHostReportSize) { + this.liveHostReportSize = liveHostReportSize; + } + + ApplicationContext appCtx; + public void setApplicationContext(ApplicationContext appCtx) throws BeansException { + this.appCtx = appCtx; + } + + /** + * Messages from the StatisticsTracker. + */ + private final static Logger logger = + Logger.getLogger(StatisticsTracker.class.getName()); + + /** + * All report types, for iteration dump at end + */ + @SuppressWarnings("unchecked") + public static final Class[] REPORTS = { + HostsReport.class, MimetypesReport.class, + ResponseCodeReport.class, SeedsReport.class, + CrawlSummaryReport.class, ProcessorsReport.class, + FrontierReport.class, SourceTagsReport.class}; + + /** + * The interval between writing progress information to log. + */ + int intervalSeconds = 20; + public int getIntervalSeconds() { + return this.intervalSeconds; + } + public void setIntervalSeconds(int interval) { + this.intervalSeconds = interval; + } + + /** + * Number of crawl-stat sample snapshots to keep for calculation + * purposes. + */ + int keepSnapshotsCount = 5; + public int getKeepSnapshotsCount() { + return this.keepSnapshotsCount; + } + public void setKeepSnapshotsCount(int count) { + this.keepSnapshotsCount = count; + } + + protected CrawlController controller; + public CrawlController getCrawlController() { + return this.controller; + } + @Autowired + public void setCrawlController(CrawlController controller) { + this.controller = controller; + } + + /** wall-clock time the crawl started */ + long crawlStartTime; + /** wall-clock time the crawl ended */ + long crawlEndTime = -1; // Until crawl ends, this value is -1. + /** wall-clock time of last pause, while pause in progres */ + long crawlPauseStarted = 0; + /** duration tally of all time spent in paused state */ + long crawlTotalPausedTime = 0; + + /** snapshots of crawl tallies and rates */ + LinkedList snapshots = new LinkedList(); + + ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + + /* + * Cumulative data + */ + /** tally sizes novel, verified (same hash), vouched (not-modified) */ + CrawledBytesHistotable crawledBytes = new CrawledBytesHistotable(); + + /** Keep track of the file types we see (mime type -> count) */ + protected Hashtable mimeTypeDistribution + = new Hashtable(); + protected Hashtable mimeTypeBytes + = new Hashtable(); + + /** Keep track of fetch status codes */ + protected Hashtable statusCodeDistribution + = new Hashtable(); + + /** Keep track of hosts. + * + * Each of these Maps are individually unsynchronized, and cannot + * be trivially synchronized with the Collections wrapper. Thus + * their synchronized access is enforced by this class. + */ + protected Map hostsDistribution = null; + protected Map hostsBytes = null; + protected Map hostsLastFinished = null; + + /** Keep track of URL counts per host per seed */ + protected + Map> sourceHostDistribution = null; + + /* Keep track of 'top' hosts for live reports */ + protected TopNSet hostsDistributionTop; + protected TopNSet hostsBytesTop; + protected TopNSet hostsLastFinishedTop; + + /** + * Record of seeds' latest actions. + */ + protected Map processedSeedsRecords; + long seedsTotal = -1; + long seedsCrawled = -1; + + public StatisticsTracker() { + + } + + boolean isRunning = false; + public boolean isRunning() { + return isRunning; + } + public void stop() { + isRunning = false; + executor.shutdownNow(); + } + + public void start() { + isRunning = true; + try { + this.sourceHostDistribution = bdb.getBigMap("sourceHostDistribution", + false, String.class, HashMap.class); + this.hostsDistribution = bdb.getBigMap("hostsDistribution", + false, String.class, LongWrapper.class); + this.hostsBytes = bdb.getBigMap("hostsBytes", false, String.class, + LongWrapper.class); + this.hostsLastFinished = bdb.getBigMap("hostsLastFinished", + false, String.class, Long.class); + this.processedSeedsRecords = bdb.getBigMap("processedSeedsRecords", + false, String.class, SeedRecord.class); + + this.hostsDistributionTop = new TopNSet(getLiveHostReportSize()); + this.hostsBytesTop = new TopNSet(getLiveHostReportSize()); + this.hostsLastFinishedTop = new TopNSet(getLiveHostReportSize()); + } catch (DatabaseException e) { + throw new IllegalStateException(e); + } + // Log the legend + this.controller.logProgressStatistics(progressStatisticsLegend()); + executor.scheduleAtFixedRate(this, 0, getIntervalSeconds(), TimeUnit.SECONDS); + } + + /** + * Start thread. Will call logActivity() at intervals specified by + * logInterval + * + */ + public void run() { + progressStatisticsEvent(); + } + + /** + * @return legend for progress-statistics lines/log + */ + public String progressStatisticsLegend() { + return " timestamp" + + " discovered " + + " queued downloaded doc/s(avg) KB/s(avg) " + + " dl-failures busy-thread mem-use-KB heap-size-KB " + + " congestion max-depth avg-depth"; + } + + /** + * Notify tracker that crawl has begun. Must be called + * outside tracker's own thread, to ensure it is noted + * before other threads start interacting with tracker. + */ + public void noteStart() { + if (this.crawlStartTime == 0) { + // Note the time the crawl starts (only if not already set) + this.crawlStartTime = System.currentTimeMillis(); + } + } + + /** + * A method for logging current crawler state. + * + * This method will be called by run() at intervals specified in + * the crawl order file. It is also invoked when pausing or + * stopping a crawl to capture the state at that point. Default behavior is + * call to {@link CrawlController#logProgressStatistics} so CrawlController + * can act on progress statistics event. + *

      + * It is recommended that for implementations of this method it be + * carefully considered if it should be synchronized in whole or in + * part + * @param e Progress statistics event. + */ + protected synchronized void progressStatisticsEvent() { + CrawlStatSnapshot snapshot = getSnapshot(); + + if (this.controller != null) { + this.controller.logProgressStatistics(snapshot.getProgressStatisticsLine()); + } + snapshots.addFirst(snapshot); + while(snapshots.size()>getKeepSnapshotsCount()) { + snapshots.removeLast(); + } + + // publish app event + appCtx.publishEvent(new StatSnapshotEvent(this,snapshot)); + + // temporary workaround for + // [ 996161 ] Fix DNSJava issues (memory) -- replace with JNDI-DNS? + // http://sourceforge.net/support/tracker.php?aid=996161 + Lookup.getDefaultCache(DClass.IN).clearCache(); + } + + public CrawlStatSnapshot getSnapshot() { + // TODO: take snapshot implementation from a spring prototype? + CrawlStatSnapshot snapshot = new CrawlStatSnapshot(); + snapshot.collect(controller,this); + return snapshot; + } + + public CrawlStatSnapshot getLastSnapshot() { + return snapshots.peek(); + } + + public long getCrawlElapsedTime() { + if (crawlStartTime == 0) { + // if no start time set yet, consider elapsed time zero + return 0; + } + if (crawlPauseStarted != 0) { + // currently paused, calculate time up to last pause + return crawlPauseStarted - crawlTotalPausedTime - crawlStartTime; + } + + // not paused, calculate total time to end or (if running) now + return ((crawlEndTime>0)?crawlEndTime:System.currentTimeMillis()) + - crawlTotalPausedTime - crawlStartTime; + } + + public void crawlPausing(String statusMessage) { + logNote("CRAWL WAITING - " + statusMessage); + } + + protected void logNote(final String note) { + this.controller.logProgressStatistics(new PaddingStringBuffer() + .append(ArchiveUtils.getLog14Date(new Date())) + .append(" ") + .append(note) + .toString()); + } + + public void crawlPaused(String statusMessage) { + crawlPauseStarted = System.currentTimeMillis(); + progressStatisticsEvent(); + logNote("CRAWL PAUSED - " + statusMessage); + } + + public void crawlResuming(String statusMessage) { + tallyCurrentPause(); + if (this.crawlStartTime == 0) { + noteStart(); + } + logNote("CRAWL RESUMED - " + statusMessage); + } + + /** + * For a current pause (if any), add paused time to total and reset + */ + protected void tallyCurrentPause() { + if (this.crawlPauseStarted > 0) { + // Ok, we managed to actually pause before resuming. + this.crawlTotalPausedTime + += (System.currentTimeMillis() - this.crawlPauseStarted); + } + this.crawlPauseStarted = 0; + } + + public void crawlEnding(String sExitMessage) { + logNote("CRAWL ENDING - " + sExitMessage); + } + + public void crawlEnded(String sExitMessage) { + // Note the time when the crawl stops. + crawlEndTime = System.currentTimeMillis(); + progressStatisticsEvent(); + logNote("CRAWL ENDED - " + sExitMessage); + } + + + public void finalTasks() { + dumpReports(); + } + + /** + * Returns how long the current crawl has been running *including* + * time paused (contrast with getCrawlElapsedTime()). + * + * @return The length of time - in msec - that this crawl has been running. + */ + public long getCrawlDuration() { + return ((crawlEndTime>0)?crawlEndTime:System.currentTimeMillis()) + - crawlStartTime; + } + + /** Returns a HashMap that contains information about distributions of + * encountered mime types. Key/value pairs represent + * mime type -> count. + *

      + * Note: All the values are wrapped with a {@link LongWrapper LongWrapper} + * @return mimeTypeDistribution + */ + public Hashtable getFileDistribution() { + return mimeTypeDistribution; + } + + + /** + * Increment a counter for a key in a given HashMap. Used for various + * aggregate data. + * + * As this is used to change Maps which depend on StatisticsTracker + * for their synchronization, this method should only be invoked + * from a a block synchronized on 'this'. + * + * @param map The HashMap + * @param key The key for the counter to be incremented, if it does not + * exist it will be added (set to 1). If null it will + * increment the counter "unknown". + */ + protected static void incrementMapCount(Map map, + String key) { + incrementMapCount(map,key,1); + } + + /** + * Increment a counter for a key in a given HashMap by an arbitrary amount. + * Used for various aggregate data. The increment amount can be negative. + * + * As this is used to change Maps which depend on StatisticsTracker + * for their synchronization, this method should only be invoked + * from a a block synchronized on 'this'. + * + * @param map + * The HashMap + * @param key + * The key for the counter to be incremented, if it does not exist + * it will be added (set to equal to increment). + * If null it will increment the counter "unknown". + * @param increment + * The amount to increment counter related to the key. + */ + protected static void incrementMapCount(Map map, + String key, long increment) { + if (key == null) { + key = "unknown"; + } + Object o = map.get(key); + if (o == null) { + // Considered normal + map.put(key, new LongWrapper(increment)); + } else if (o instanceof LongWrapper) { + LongWrapper lw = (LongWrapper)o; + lw.longValue += increment; + } else { + // Abnormal + logger.severe("Resetting " + key + ": Expected LongWrapper but got " + + o.getClass().getName()); + map.put(key, new LongWrapper(increment)); + } + } + + /** + * Sort the entries of the given HashMap in descending order by their + * values, which must be longs wrapped with LongWrapper. + *

      + * Elements are sorted by value from largest to smallest. Equal values are + * sorted in an arbitrary, but consistent manner by their keys. Only items + * with identical value and key are considered equal. + * + * If the passed-in map requires access to be synchronized, the caller + * should ensure this synchronization. + * + * @param mapOfLongWrapperValues + * Assumes values are wrapped with LongWrapper. + * @return a sorted set containing the same elements as the map. + */ + public TreeMap getReverseSortedCopy( + final Map mapOfLongWrapperValues) { + TreeMap sortedMap = + new TreeMap(new Comparator() { + public int compare(String e1, String e2) { + long firstVal = mapOfLongWrapperValues.get(e1). + longValue; + long secondVal = mapOfLongWrapperValues.get(e2). + longValue; + if (firstVal < secondVal) { + return 1; + } + if (secondVal < firstVal) { + return -1; + } + // If the values are the same, sort by keys. + return e1.compareTo(e2); + } + }); + try { + sortedMap.putAll(mapOfLongWrapperValues); + } catch (UnsupportedOperationException e) { + Iterator i = mapOfLongWrapperValues.keySet().iterator(); + for (;i.hasNext();) { + // Ok. Try doing it the slow way then. + String key = i.next(); + sortedMap.put(key, mapOfLongWrapperValues.get(key)); + } + } + return sortedMap; + } + + /** + * Return a HashMap representing the distribution of status codes for + * successfully fetched curis, as represented by a hashmap where key -> + * val represents (string)code -> (integer)count. + * + * Note: All the values are wrapped with a + * {@link LongWrapper LongWrapper} + * + * @return statusCodeDistribution + */ + public Hashtable getStatusCodeDistribution() { + return statusCodeDistribution; + } + + /** + * Returns the time (in millisec) when a URI belonging to a given host was + * last finished processing. + * + * @param host The host to look up time of last completed URI. + * @return Returns the time (in millisec) when a URI belonging to a given + * host was last finished processing. If no URI has been completed for host + * -1 will be returned. + */ + public long getHostLastFinished(String host){ + Long l = null; + synchronized(hostsLastFinished){ + l = (Long)hostsLastFinished.get(host); + } + return (l != null)? l.longValue(): -1; + } + + /** + * Returns the accumulated number of bytes downloaded from a given host. + * @param host name of the host + * @return the accumulated number of bytes downloaded from a given host + */ + public long getBytesPerHost(String host){ + synchronized(hostsBytes){ + return getReportValue(hostsBytes, host); + } + } + + /** + * Returns the accumulated number of bytes from files of a given file type. + * @param filetype Filetype to check. + * @return the accumulated number of bytes from files of a given mime type + */ + public long getBytesPerFileType(String filetype){ + return getReportValue(mimeTypeBytes, filetype); + } + + /** + * Get the total number of ToeThreads (sleeping and active) + * + * @return The total number of ToeThreads + */ + public int threadCount() { + return this.controller != null? controller.getToeCount(): 0; + } + + public String crawledBytesSummary() { + return crawledBytes.summary(); + } + + /** + * If the curi is a seed, we update the processedSeeds table. + * + * @param curi The CrawlURI that may be a seed. + * @param disposition The disposition of the CrawlURI. + */ + private void handleSeed(CrawlURI curi, String disposition) { + if(curi.isSeed()){ + SeedRecord sr = new SeedRecord(curi, disposition); + processedSeedsRecords.put(sr.getUri(), sr); + } + } + + public void crawledURISuccessful(CrawlURI curi) { + handleSeed(curi,"Seed successfully crawled"); + // save crawled bytes tally + crawledBytes.accumulate(curi); + + // Save status codes + incrementMapCount(statusCodeDistribution, + Integer.toString(curi.getFetchStatus())); + + // Save mime types + String mime = MimetypeUtils.truncate(curi.getContentType()); + incrementMapCount(mimeTypeDistribution, mime); + incrementMapCount(mimeTypeBytes, mime, curi.getContentSize()); + + // Save hosts stats. + ServerCache sc = serverCache; + saveHostStats((curi.getFetchStatus() == 1)? "dns:": + ServerCacheUtil.getHostFor(sc, curi.getUURI()).getHostName(), + curi.getContentSize()); + + if (curi.getData().containsKey(A_SOURCE_TAG)) { + saveSourceStats((String)curi.getData().get(A_SOURCE_TAG), + ServerCacheUtil.getHostFor(sc, curi.getUURI()). + getHostName()); + } + } + + protected void saveSourceStats(String source, String hostname) { + synchronized(sourceHostDistribution) { + HashMap hostUriCount = + sourceHostDistribution.get(source); + if (hostUriCount == null) { + hostUriCount = new HashMap(); + } + // TODO: Dan suggests we don't need a hashtable value. Might + // be faster if we went without. Could just have keys of: + // seed | host (concatenated as string) + // and values of: + // #urls + incrementMapCount(hostUriCount, hostname); + sourceHostDistribution.put(source, hostUriCount); + } + } + + protected void saveHostStats(String hostname, long size) { + synchronized(hostsDistribution){ + incrementMapCount(hostsDistribution, hostname); + hostsDistributionTop.update( + hostname, getReportValue(hostsDistribution, hostname)); + } + synchronized(hostsBytes){ + incrementMapCount(hostsBytes, hostname, size); + hostsBytesTop.update(hostname, + getReportValue(hostsBytes, hostname)); + } + synchronized(hostsLastFinished){ + long time = new Long(System.currentTimeMillis()); + hostsLastFinished.put(hostname, time); + hostsLastFinishedTop.update(hostname, time); + } + } + + public void crawledURINeedRetry(CrawlURI curi) { + handleSeed(curi,"Failed to crawl seed, will retry"); + } + + public void crawledURIDisregard(CrawlURI curi) { + handleSeed(curi,"Seed was disregarded"); + } + + public void crawledURIFailure(CrawlURI curi) { + handleSeed(curi,"Failed to crawl seed"); + } + + /** + * Get a seed iterator for the job being monitored. + * + * Note: This iterator will iterate over a list of strings not + * UURIs like the Scope seed iterator. The strings are equal to the URIs' + * getURIString() values. + * @return the seed iterator + * FIXME: Consider using TransformingIterator here + */ + public Iterator getSeedsIterator() { + List seedsCopy = new Vector(); + Iterator i = seeds.seedsIterator(); + while (i.hasNext()) { + seedsCopy.add(i.next().toString()); + } + return seedsCopy.iterator(); + } + + public Iterator getSeedRecordsSortedByStatusCode() { + return getSeedRecordsSortedByStatusCode(getSeedsIterator()); + } + + protected Iterator getSeedRecordsSortedByStatusCode( + Iterator i) { + TreeSet sortedSet = + new TreeSet(new Comparator() { + public int compare(SeedRecord sr1, SeedRecord sr2) { + int code1 = sr1.getStatusCode(); + int code2 = sr2.getStatusCode(); + if (code1 == code2) { + // If the values are equal, sort by URIs. + return sr1.getUri().compareTo(sr2.getUri()); + } + // mirror and shift the nubmer line so as to + // place zero at the beginning, then all negatives + // in order of ascending absolute value, then all + // positives descending + code1 = -code1 - Integer.MAX_VALUE; + code2 = -code2 - Integer.MAX_VALUE; + + return new Integer(code1).compareTo(new Integer(code2)); + } + }); + while (i.hasNext()) { + String seed = i.next(); + SeedRecord sr = (SeedRecord) processedSeedsRecords.get(seed); + if(sr==null) { + sr = new SeedRecord(seed,"Seed has not been processed"); + processedSeedsRecords.put(seed,sr); + } + sortedSet.add(sr); + } + return sortedSet.iterator(); + } + + /** + * Return a copy of the hosts distribution in reverse-sorted (largest first) + * order. + * + * @return SortedMap of hosts distribution + */ + public SortedMap getReverseSortedHostCounts( + Map hostCounts) { + synchronized(hostCounts){ + return getReverseSortedCopy(hostCounts); + } + } + + /** + * Return a copy of the hosts distribution in reverse-sorted + * (largest first) order. + * @return SortedMap of hosts distribution + */ + public SortedMap getReverseSortedHostsDistribution() { + synchronized(hostsDistribution){ + return getReverseSortedCopy(hostsDistribution); + } + } + + protected void writeReportFile(Class reportClass) { + Report r; + try { + r = reportClass.newInstance(); + } catch (Exception e) { + throw new RuntimeException(e); + } + r.setStats(this); + File f = new File(getReportsDir().getFile(), r.getFilename()); + try { + PrintWriter bw = new PrintWriter(new FileWriter(f)); + r.write(bw); + bw.close(); + addToManifest(f.getAbsolutePath(), + CrawlerLoggerModule.MANIFEST_REPORT_FILE, true); + } catch (IOException e) { + logger.log(Level.SEVERE, "Unable to write " + f.getAbsolutePath() + + " at the end of crawl.", e); + } + logger.info("wrote report: " + f.getAbsolutePath()); + } + + protected void addToManifest(String absolutePath, char manifest_report_file, boolean b) { + // TODO Auto-generated method stub + + } + + /** + * Run the reports. + */ + public void dumpReports() { + // TODO: sooner than here! Add all files mentioned in the crawl + // order to the manifest set. + //controller.addOrderToManifest(); + + for(Class reportClass : REPORTS) { + writeReportFile(reportClass); + } + } + + public void crawlCheckpoint(/*StateProvider*/ Object def, File cpDir) throws Exception { + // CrawlController is managing the checkpointing of this object. + logNote("CRAWL CHECKPOINTING TO " + cpDir.toString()); + } + + private long getReportValue(Map map, String key) { + if (key == null) { + return -1; + } + Object o = map.get(key); + if (o == null) { + return -2; + } + if (!(o instanceof LongWrapper)) { + throw new IllegalStateException("Expected LongWrapper but got " + + o.getClass() + " for " + key); + } + return ((LongWrapper)o).longValue; + } + + public void onApplicationEvent(ApplicationEvent event) { + if(event instanceof CrawlStateEvent) { + CrawlStateEvent event1 = (CrawlStateEvent)event; + switch(event1.getState()) { + case PAUSED: + this.crawlPaused(event1.getMessage()); + break; + case RUNNING: + this.crawlResuming(event1.getMessage()); + break; + case PAUSING: + this.crawlPausing(event1.getMessage()); + break; + case STOPPING: + this.crawlEnding(event1.getMessage()); + break; + case FINISHED: + this.crawlEnded(event1.getMessage()); + break; + case PREPARING: + this.crawlResuming(event1.getMessage()); + break; + default: + throw new RuntimeException("Unknown state: " + event1.getState()); + } + } + + if(event instanceof CrawlURIDispositionEvent) { + CrawlURIDispositionEvent dvent = (CrawlURIDispositionEvent)event; + switch(dvent.getDisposition()) { + case SUCCEEDED: + this.crawledURISuccessful(dvent.getCrawlURI()); + break; + case FAILED: + this.crawledURIFailure(dvent.getCrawlURI()); + break; + case DISREGARDED: + this.crawledURIDisregard(dvent.getCrawlURI()); + break; + case DEFERRED_FOR_RETRY: + this.crawledURINeedRetry(dvent.getCrawlURI()); + break; + default: + throw new RuntimeException("Unknown disposition: " + dvent.getDisposition()); + } + } + } + + public void tallySeeds() { + seedsTotal = 0; + seedsCrawled = 0; + for (Iterator i = getSeedsIterator();i.hasNext();) { + SeedRecord sr = processedSeedsRecords.get(i.next()); + seedsTotal++; + if((sr.getStatusCode() > 0)) { + seedsCrawled++; + } + } + } +} diff --git a/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java b/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java new file mode 100644 index 00000000..69957bb2 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java @@ -0,0 +1,81 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.restlet; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.Reader; +import java.io.StringReader; +import java.io.StringWriter; +import java.io.Writer; + +import org.apache.commons.lang.StringEscapeUtils; +import org.restlet.data.CharacterSet; +import org.restlet.data.MediaType; +import org.restlet.data.Reference; +import org.restlet.resource.CharacterRepresentation; +import org.restlet.resource.FileRepresentation; + +/** + * Representation wrapping a FileRepresentation, displaying its contents + * in a TextArea for editting. + * + * @contributor gojomo + */ +public class EditRepresentation extends CharacterRepresentation { + FileRepresentation fileRepresentation; + EnhDirectoryResource dirResource; + + public EditRepresentation(FileRepresentation representation, EnhDirectoryResource resource) { + super(MediaType.TEXT_HTML); + fileRepresentation = representation; + dirResource = resource; + // TODO: remove if not necessary in future? + setCharacterSet(CharacterSet.UTF_8); + } + + @Override + public Reader getReader() throws IOException { + StringWriter writer = new StringWriter((int)fileRepresentation.getSize()+100); + write(writer); + return new StringReader(writer.toString()); + } + + @Override + public void write(Writer writer) throws IOException { + PrintWriter pw = new PrintWriter(writer); + Flash.renderFlashesHTML(pw, dirResource.getRequest()); + pw.println("

      "); + pw.println(""); + pw.println(fileRepresentation.getFile()); + Reference viewRef = dirResource.getRequest().getOriginalRef().clone(); + viewRef.setQuery(null); + pw.println("view"); + pw.println("
      "); + pw.println("
      "); + pw.close(); + } + + public FileRepresentation getFileRepresentation() { + return fileRepresentation; + } +} diff --git a/engine/src/main/java/org/archive/crawler/restlet/EngineApplication.java b/engine/src/main/java/org/archive/crawler/restlet/EngineApplication.java new file mode 100644 index 00000000..437ca314 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/restlet/EngineApplication.java @@ -0,0 +1,90 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.restlet; + +import java.io.IOException; + +import org.archive.crawler.framework.Engine; +import org.restlet.Application; +import org.restlet.Directory; +import org.restlet.Restlet; +import org.restlet.Router; +import org.restlet.data.MediaType; +import org.restlet.data.Reference; +import org.restlet.data.Request; +import org.restlet.util.Template; + +/** + * Restlet Application for a Heritrix crawl 'Engine', which is aware of + * local job configurations/directories and can assemble/launch/monitor/ + * manage crawls. + * + * @contributor gojomo + */ +public class EngineApplication extends Application { + Engine engine; + public EngineApplication(Engine engine) { + this.engine = engine; + getMetadataService().addExtension("log", MediaType.TEXT_PLAIN ); + getMetadataService().addExtension("cxml", MediaType.APPLICATION_XML ); + } + + public synchronized Restlet createRoot() { + Router router = new Router(getContext()); + + router.attach("/engine",EngineResource.class) + .setMatchingMode(Template.MODE_EQUALS); + router.attach("/engine/",EngineResource.class) + .setMatchingMode(Template.MODE_EQUALS); + + Directory alljobsdir = new Directory( + getContext(), + engine.getJobsDir().toURI().toString()); + alljobsdir.setListingAllowed(true); + router.attach("/engine/jobsdir",alljobsdir); + + + EnhDirectory jobdir = new EnhDirectory( + getContext(), + engine.getJobsDir().toURI().toString() /*TODO: changeme*/) { + @Override + Reference determineRootRef(Request request) { + try { + return new Reference( + EngineApplication.this.getEngine() + .getJob((String)request.getAttributes().get("job")) + .getJobDir().getCanonicalFile().toURI().toString()); + } catch (IOException e) { + throw new RuntimeException(e); + } + }}; + jobdir.setListingAllowed(true); + jobdir.setModifiable(true); + jobdir.setEditFilter(JobResource.EDIT_FILTER); + router.attach("/engine/job/{job}/jobdir",jobdir); + router.attach("/engine/job/{job}",JobResource.class); + + return router; + } + + public Engine getEngine() { + return engine; + } +} diff --git a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java new file mode 100644 index 00000000..67979cee --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java @@ -0,0 +1,116 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.restlet; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.Writer; +import java.util.ArrayList; +import java.util.Collections; + +import org.archive.crawler.framework.CrawlJob; +import org.archive.crawler.framework.Engine; +import org.restlet.Context; +import org.restlet.data.CharacterSet; +import org.restlet.data.Form; +import org.restlet.data.MediaType; +import org.restlet.data.Request; +import org.restlet.data.Response; +import org.restlet.resource.Representation; +import org.restlet.resource.Resource; +import org.restlet.resource.ResourceException; +import org.restlet.resource.Variant; +import org.restlet.resource.WriterRepresentation; + +/** + * Restlet Resource representing an Engine that may be used + * to assemble, launch, monitor, and manage crawls. + * + * @contributor gojomo + */ +public class EngineResource extends Resource { + + public EngineResource(Context ctx, Request req, Response res) { + super(ctx, req, res); + setModifiable(true); + getVariants().add(new Variant(MediaType.TEXT_HTML)); + } + + public Representation represent(Variant variant) throws ResourceException { + Representation representation = new WriterRepresentation( + MediaType.TEXT_HTML) { + public void write(Writer writer) throws IOException { + EngineResource.this.writeHtml(writer); + } + }; + // TODO: remove if not necessary in future? + representation.setCharacterSet(CharacterSet.UTF_8); + return representation; + } + + @Override + public void acceptRepresentation(Representation entity) throws ResourceException { + Form form = getRequest().getEntityAsForm(); + String action = form.getFirstValue("action"); + if("rescan".equals(action)) { + getEngine().findJobConfigs(); + } + // default: redirect to GET self + getResponse().redirectSeeOther(getRequest().getOriginalRef()); + } + + protected void writeHtml(Writer writer) { + Engine engine = getEngine(); + String engineTitle = "Heritrix Engine "+engine.getHeritrixVersion(); + String baseRef = getRequest().getResourceRef().getBaseRef().toString(); + if(!baseRef.endsWith("/")) { + baseRef += "/"; + } + PrintWriter pw = new PrintWriter(writer); + pw.println(""+engineTitle+""); + pw.println(""); + pw.println(""); + pw.println("

      "+engineTitle+"

      "); + + pw.println("Memory: "); + pw.println(engine.heapReport()); + + pw.println("

      Browse Jobs Directory

      "); + + ArrayList jobs = new ArrayList(); + jobs.addAll(engine.getJobConfigs().values()); + + pw.println("

      Job Configs ("+jobs.size()+")"); + pw.println(""); + pw.println("

      "); + Collections.sort(jobs); + for(CrawlJob cj: jobs) { + pw.println("
    1. "); + cj.writeHtmlTo(pw,"job/"); + pw.println("
      2. "); + } + pw.println(""); + pw.flush(); + } + + protected Engine getEngine() { + return ((EngineApplication)getApplication()).getEngine(); + } +} diff --git a/engine/src/main/java/org/archive/crawler/restlet/EnhDirectory.java b/engine/src/main/java/org/archive/crawler/restlet/EnhDirectory.java new file mode 100644 index 00000000..b816feed --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/restlet/EnhDirectory.java @@ -0,0 +1,91 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.restlet; + +import java.io.File; +import java.io.IOException; +import java.util.logging.Level; + +import org.apache.commons.io.filefilter.FileFilterUtils; +import org.apache.commons.io.filefilter.IOFileFilter; +import org.restlet.Context; +import org.restlet.Handler; +import org.restlet.data.Reference; +import org.restlet.data.Request; +import org.restlet.data.Response; + +/** + * Enhanced version of Restlet Directory, which allows the local + * filesystem directory to be determined dynamically based on the + * request details. Also, via use of an EnhDirectoryResource, adds + * other capabilities (editting, etc.). + * + * @contributor gojomo + */ +public abstract class EnhDirectory extends org.restlet.Directory { + IOFileFilter editFilter = FileFilterUtils.falseFileFilter(); + IOFileFilter pageFilter = FileFilterUtils.falseFileFilter(); + IOFileFilter tailFilter = FileFilterUtils.falseFileFilter(); + + public EnhDirectory(Context context, Reference rootLocalReference) { + super(context, rootLocalReference); + // TODO Auto-generated constructor stub + } + + public EnhDirectory(Context context, String rootUri) { + super(context, rootUri); + // TODO Auto-generated constructor stub + } + + @Override + public Handler findTarget(Request request, Response response) { + Handler retVal; + synchronized(this) { + Reference oldRef = getRootRef(); + setRootRef(determineRootRef(request)); + try { + retVal = new EnhDirectoryResource(this, request, response); + } catch (IOException ioe) { + getLogger().log(Level.WARNING, + "Unable to find the directory's resource", ioe); + retVal = null; + } + setRootRef(oldRef); + } + return retVal; + } + + abstract Reference determineRootRef(Request request); + + public boolean allowsEdit(File file) { + return editFilter.accept(file); + } + + public void setEditFilter(IOFileFilter fileFilter) { + editFilter = fileFilter; + } + + public boolean allowsPaging(File file) { + // TODO: limit? + return true; + } + + +} diff --git a/engine/src/main/java/org/archive/crawler/restlet/EnhDirectoryResource.java b/engine/src/main/java/org/archive/crawler/restlet/EnhDirectoryResource.java new file mode 100644 index 00000000..df4ef750 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/restlet/EnhDirectoryResource.java @@ -0,0 +1,140 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.archive.crawler.restlet; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.List; +import java.util.ListIterator; + +import org.apache.commons.io.FileUtils; +import org.restlet.data.Form; +import org.restlet.data.Reference; +import org.restlet.data.Request; +import org.restlet.data.Response; +import org.restlet.resource.FileRepresentation; +import org.restlet.resource.Representation; +import org.restlet.resource.ResourceException; +import org.restlet.resource.Variant; + +import com.noelios.restlet.local.DirectoryResource; + +/** + * Enhanced version of Restlet DirectoryResource, adding ability to + * edit some files. + * + * @contributor gojomo + */ +public class EnhDirectoryResource extends DirectoryResource { + + public EnhDirectoryResource(EnhDirectory directory, Request request, Response response) throws IOException { + super(directory, request, response); + } + + /** + * Add EditRepresentation as a variant when appropriate. + * + * @see com.noelios.restlet.local.DirectoryResource#getVariants() + */ + @Override + public List getVariants() { + List variants = super.getVariants(); + Form f = getRequest().getResourceRef().getQueryAsForm(); + String format = f.getFirstValue("format"); + if("textedit".equals(format)) { + if(variants.isEmpty()) { + // create empty placeholder file if appropriate + try { + File file = new File(new URI(getTargetUri())); + if(getEnhDirectory().allowsEdit(file)) { + file.createNewFile(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + variants = super.getVariants(); + } + // wrap FileRepresentations in EditRepresentations + ListIterator iter = variants.listIterator(); + while(iter.hasNext()) { + Variant v = iter.next(); + if(v instanceof FileRepresentation) { + File file = ((FileRepresentation)v).getFile(); + if(getEnhDirectory().allowsEdit(file)) { + iter.remove(); + iter.add(new EditRepresentation((FileRepresentation)v,this)); + }; + } + } + } else if("paged".equals(format)) { + ListIterator iter = variants.listIterator(); + while(iter.hasNext()) { + Variant v = iter.next(); + if(v instanceof FileRepresentation) { + File file = ((FileRepresentation)v).getFile(); + if(getEnhDirectory().allowsPaging(file)) { + iter.remove(); + iter.add(new PagedRepresentation(( + FileRepresentation)v, + this, + f.getFirstValue("pos"), + f.getFirstValue("lines"), + f.getFirstValue("reverse"))); + }; + } + } + } + + return variants; + } + + protected EnhDirectory getEnhDirectory() { + return (EnhDirectory)getDirectory(); + } + + /** + * Accept a POST used to edit or create a file. + * + * @see org.restlet.resource.Resource#acceptRepresentation(org.restlet.resource.Representation) + */ + public void acceptRepresentation(Representation entity) + throws ResourceException { + // TODO: only allowPost on valid targets + Form form = getRequest().getEntityAsForm(); + String newContents = form.getFirstValue("contents"); + // TODO: defensive + EditRepresentation er = (EditRepresentation) getVariants().get(0); + File file = er.getFileRepresentation().getFile(); + try { + FileUtils.writeStringToFile(file, newContents); + Flash.addFlash(getResponse(), "file updated"); + } catch (IOException e) { + // TODO report error somehow + e.printStackTrace(); + } + // redirect to view version + Reference ref = getRequest().getOriginalRef().clone(); + /// ref.setQuery(null); + getResponse().redirectSeeOther(ref); + + } +} diff --git a/engine/src/main/java/org/archive/crawler/restlet/Flash.java b/engine/src/main/java/org/archive/crawler/restlet/Flash.java new file mode 100644 index 00000000..a043e838 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/restlet/Flash.java @@ -0,0 +1,135 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.restlet; + +import java.io.PrintWriter; +import java.io.Writer; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.commons.lang.math.RandomUtils; +import org.restlet.data.Cookie; +import org.restlet.data.CookieSetting; +import org.restlet.data.Request; +import org.restlet.data.Response; +import org.restlet.util.Series; + +/** + * Utility for including a brief last-action or background-action + * message on web responses. + * + * @contributor gojomo + * + */ +public class Flash { + /** usual types */ + public enum Kind {ACK, NACK, ADVISORY} + + static long nextdrop = RandomUtils.nextLong(); + static Map dropboxes = new LinkedHashMap() { + private static final long serialVersionUID = 1L; + @Override + protected boolean removeEldestEntry(Entry eldest) { + return size()>100; + } + + }; + + public static void addFlash(Response response, String message) { + addFlash(response, message, Kind.ACK); + } + + public static void addFlash(Response response, String message, Kind kind) { + dropboxes.put(nextdrop,new Flash(message, kind)); + Series cookies = response.getCookieSettings(); + cookies.add(new CookieSetting("flashdrop",Long.toString(nextdrop))); + nextdrop++; + } + + public static List getFlashes(Request request) { + List flashes = new LinkedList(); + Series cookies = request.getCookies(); + String dropbox = cookies.getFirstValue("flashdrop"); + if(dropbox!=null) { + Flash flash = dropboxes.remove(Long.parseLong(dropbox)); + if(flash!=null) { + flashes.add(flash); + } + } + return flashes; + } + + public static void renderFlashesHTML(Writer writer, Request request) { + PrintWriter pw = new PrintWriter(writer); + for(Flash flash : getFlashes(request)) { + pw.println("
      "); + pw.println(flash.getMessage()); + pw.println("
      "); + } + pw.flush(); + } + + + /** kind of flash, ACK NACK or ADVISORY */ + protected Kind kind; + /** the message to show, if any */ + protected String message; + + /** + * Create an ACK flash of default styling with the given message. + * + * @param message + */ + public Flash(String message) { + this(message, Kind.ACK); + } + + /** + * Create a Flash of the given kind, message with default styling. + * + * @param kind + * @param message + */ + public Flash(String message, Kind kind) { + this.kind = kind; + this.message = message; + } + + /** + * Indicate whether the Flash should persist. The usual and + * default case is that a Flash displays once and then expires. + * + * @return boolean whether to discard Flash + */ + public boolean isExpired() { + return true; + } + + public String getMessage() { + return this.message; + } + + public Kind getKind() { + return this.kind; + } +} diff --git a/engine/src/main/java/org/archive/crawler/restlet/JobResource.java b/engine/src/main/java/org/archive/crawler/restlet/JobResource.java new file mode 100644 index 00000000..d2b9df6a --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/restlet/JobResource.java @@ -0,0 +1,364 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.restlet; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.Writer; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; + +import org.apache.commons.io.filefilter.IOFileFilter; +import org.apache.commons.lang.StringEscapeUtils; +import org.archive.crawler.framework.CrawlJob; +import org.archive.crawler.framework.Engine; +import org.archive.spring.ConfigPath; +import org.archive.util.ArchiveUtils; +import org.archive.util.FileUtils; +import org.restlet.Context; +import org.restlet.data.CharacterSet; +import org.restlet.data.Form; +import org.restlet.data.MediaType; +import org.restlet.data.Request; +import org.restlet.data.Response; +import org.restlet.data.Status; +import org.restlet.resource.Representation; +import org.restlet.resource.Resource; +import org.restlet.resource.ResourceException; +import org.restlet.resource.Variant; +import org.restlet.resource.WriterRepresentation; + +/** + * Restlet Resource representing a single local CrawlJob inside an + * Engine. + * + * @contributor gojomo + */ +public class JobResource extends Resource { + public static final IOFileFilter EDIT_FILTER = + FileUtils.getRegexFileFilter(".*\\.((c?xml)|(txt))$"); + + CrawlJob cj; + + public JobResource(Context ctx, Request req, Response res) throws ResourceException { + super(ctx, req, res); + setModifiable(true); + getVariants().add(new Variant(MediaType.TEXT_HTML)); + cj = getEngine().getJob((String)req.getAttributes().get("job")); + if(cj==null) { + throw new ResourceException(404); + } + } + + public Representation represent(Variant variant) throws ResourceException { + Representation representation = new WriterRepresentation( + MediaType.TEXT_HTML) { + public void write(Writer writer) throws IOException { + JobResource.this.writeHtml(writer); + } + }; + // TODO: remove if not necessary in future? + representation.setCharacterSet(CharacterSet.UTF_8); + return representation; + } + + protected void writeHtml(Writer writer) { + PrintWriter pw = new PrintWriter(writer); + String jobTitle = "Job "+cj.getShortName(); + String baseRef = getRequest().getResourceRef().getBaseRef().toString(); + if(!baseRef.endsWith("/")) { + baseRef += "/"; + } + // TODO: replace with use a templating system (FreeMarker?) + pw.println(""+jobTitle+""); + pw.println(""); + pw.println(""); + pw.print("

      Job "+cj.getShortName()+" ("); + + pw.print(cj.getLaunchCount() + " launches"); + if(cj.getLastLaunch()!=null) { + long ago = System.currentTimeMillis() - cj.getLastLaunch().getMillis(); + pw.println(", last "+ArchiveUtils.formatMillisecondsToConventional(ago, 2)+" ago)"); + } + pw.println(")

      "); + + + // button controls + pw.println("
      "); + // PREP, LAUNCH + pw.print(""); + pw.print(" - "); + + // PAUSE, UNPAUSE, CHECKPOINT + pw.println(""); + pw.println(""); + pw.println(" - "); + + + // TERMINATE, RESET + pw.println(""); + pw.println("
      "); + + pw.println("
      "); + + // configuration + pw.println("configuration: "); + printLinkedIfInJobDirectory(pw, cj.getPrimaryConfig()); + for(File f : cj.getImportedConfigs(cj.getPrimaryConfig())) { + pw.println("imported: "); + printLinkedIfInJobDirectory(pw,f); + } + +// if(cj.isXmlOk()) { +// pw.println("cxml ok
      "); +// if(cj.isContainerOk()) { +// pw.println("container ok
      "); +// if(cj.isContainerValidated()) { +// pw.println("config valid
      "); +// } else { +// pw.println("CONFIG INVALID
      "); +// } +// } else { +// pw.println("CONTAINER BAD
      "); +// } +// }else { +// // pw.println("XML NOT WELL-FORMED
      "); +// } + + pw.println("

      Job Log "); + pw.println("(more)"); + pw.println("

      "); + pw.println("
      "); + if(cj.getJobLog().exists()) { + try { + List logLines = new LinkedList(); + FileUtils.pagedLines(cj.getJobLog(), -1, -5, logLines); + Collections.reverse(logLines); + for(String line : logLines) { + pw.print("

      "); + StringEscapeUtils.escapeHtml(pw,line); + pw.print("

      "); + } + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + } + pw.println("
      "); + + + if(!cj.isContainerOk()) { + pw.println("

      Unbuilt Job

      "); + } else if(cj.isRunning()) { + pw.println("

      Active Job: "+cj.getCrawlController().getState()+"

      "); + } else if(cj.isLaunchable()){ + pw.println("

      Ready Job

      "); + } else { + pw.println("

      Finished Job: "+cj.getCrawlController().getCrawlExitStatus()+"

      "); + } + + if(cj.isContainerOk()) { + pw.println("Totals
        "); + pw.println(cj.uriTotalsReport()); + pw.println("
        "); + pw.println(cj.sizeTotalsReport()); + + pw.println("
      Alerts
        "); + pw.println(cj.getAlertCount()==0 ? "none" : cj.getAlertCount()); + if(cj.getAlertCount()>0) { + pw.println("tail alert log..."); + } + + pw.println("
      Rates
        "); + pw.println(cj.rateReport()); + + pw.println("
      Load
        "); + pw.println(cj.loadReport()); + + pw.println("
      Elapsed
        "); + pw.println(cj.elapsedReport()); + + pw.println("
      Threads
        "); + pw.println(cj.threadReport()); + + pw.println("
      Frontier
        "); + pw.println(cj.frontierReport()); + + pw.println("
      Memory
        "); + pw.println(getEngine().heapReport()); + + if(cj.isRunning() || (cj.isContainerOk() && !cj.isLaunchable())) { + // show crawl log for running or finished crawls + pw.println("

      Crawl Log"); + pw.println("(more)"); + pw.println("

      "); + pw.println("
      ");
+                try {
+                    List logLines = new LinkedList();
+                    FileUtils.pagedLines(
+                            cj.getCrawlController().getLoggerModule().getCrawlLogPath().getFile(),
+                            -1, 
+                            -10, 
+                            logLines);
+                    Collections.reverse(logLines);
+                    for(String line : logLines) {
+                        StringEscapeUtils.escapeHtml(pw,line);
+                        pw.println();
+                    }
+                } catch (IOException ioe) {
+                    throw new RuntimeException(ioe); 
+                }
+                pw.println("
      "); + } + + } + + pw.println("

      Files

      "); + pw.println("

      Browse Job Directory

      "); + // specific paths from wired context + pw.println("

      Configuration-referenced Paths

      "); + if(cj.getConfigPaths().isEmpty()) { + pw.println("build the job to discover referenced paths"); + } else { + pw.println("
      "); + for(ConfigPath cp : cj.getConfigPaths().values()) { + pw.println("
      "+cp.getName()+"
      "); + pw.println("
      "); + printLinkedIfInJobDirectory(pw, cp.getFile()); + pw.println("
      "); + } + } + pw.println("
      "); + pw.println("
      "); + pw.println( + "
      Copy job to " + + "" + + "" + + "
      "); + pw.println("
      "); + pw.close(); + } + + /** + * Print the given File path, but only provide view/edit link if + * path is within job directory. + * + * @param pw PrintWriter + * @param f File + */ + protected void printLinkedIfInJobDirectory(PrintWriter pw, File f) { + String jobDirRelative = cj.jobDirRelativePath(f); + if(jobDirRelative==null) { + pw.println(f); + return; + } + pw.println("" + + f +""); + if(EDIT_FILTER.accept(f)) { + pw.println("[edit]
      "); + } + } + + protected Engine getEngine() { + return ((EngineApplication)getApplication()).getEngine(); + } + + @Override + public void acceptRepresentation(Representation entity) throws ResourceException { + // copy op? + Form form = getRequest().getEntityAsForm(); + String copyTo = form.getFirstValue("copyTo"); + if(copyTo!=null) { + copyJob(copyTo,"on".equals(form.getFirstValue("asProfile"))); + return; + } + String action = form.getFirstValue("action"); + if("launch".equals(action)) { + cj.launch(); + } else if("checkXML".equals(action)) { + cj.checkXML(); + } else if("instantiate".equals(action)) { + cj.instantiateContainer(); + } else if("build".equals(action)||"validate".equals(action)) { + cj.validateConfiguration(); + } else if("discard".equals(action)) { + cj.reset(); + } else if("pause".equals(action)) { + cj.getCrawlController().requestCrawlPause(); + } else if("unpause".equals(action)) { + cj.getCrawlController().requestCrawlResume(); + } else if("terminate".equals(action)) { + cj.terminate(); + } + // default: redirect to GET self + getResponse().redirectSeeOther(getRequest().getOriginalRef()); + } + + protected void copyJob(String copyTo, boolean asProfile) throws ResourceException { + try { + getEngine().copy(cj, copyTo, asProfile); + } catch (IOException e) { + throw new ResourceException(Status.CLIENT_ERROR_CONFLICT,e); + } + // redirect to destination job page + getResponse().redirectSeeOther(copyTo); + } + + +} diff --git a/engine/src/main/java/org/archive/crawler/restlet/PagedRepresentation.java b/engine/src/main/java/org/archive/crawler/restlet/PagedRepresentation.java new file mode 100644 index 00000000..e73ed696 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/restlet/PagedRepresentation.java @@ -0,0 +1,269 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.restlet; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.Reader; +import java.io.StringReader; +import java.io.StringWriter; +import java.io.Writer; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; + +import org.apache.commons.lang.StringEscapeUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.math.LongRange; +import org.archive.util.FileUtils; +import org.restlet.data.CharacterSet; +import org.restlet.data.Form; +import org.restlet.data.MediaType; +import org.restlet.data.Reference; +import org.restlet.resource.CharacterRepresentation; +import org.restlet.resource.FileRepresentation; + +/** + * Representation wrapping a FileRepresentation, displaying its contents + * in batches of lines at a time, with forward and backward navigation. + * + * @contributor gojomo + */ +public class PagedRepresentation extends CharacterRepresentation { + // passed-in at construction + /** wrapped FileRepresentation **/ + FileRepresentation fileRepresentation; + /** wrapped EnhDirectoryResource; used to formulate self-links **/ + EnhDirectoryResource dirResource; + + /** position in file around which to fetch lines **/ + long position; + /** desired line count; negative to go back from position; default 128 **/ + int lineCount; + /** whether to display lines in reversed order (latest first) **/ + boolean reversedOrder; + + // created when file is scanned + /** text lines **/ + List lines; + /** position range [start-of-first-line, past-end-of-last-line] in file **/ + LongRange range; + /** File **/ + File file; + // TODO: maybe, freeze length for more consistent display of growing files + // (now, as length/%/bumper are written after lines retrieved, they + // sometimes are indicative the file has grown before the page is + // even rendered) + + public PagedRepresentation(FileRepresentation representation, + EnhDirectoryResource resource, String pos, String lines, + String reverse) { + super(MediaType.TEXT_HTML); + fileRepresentation = representation; + dirResource = resource; + + position = StringUtils.isBlank(pos) ? 0 : Long.parseLong(pos); + lineCount = StringUtils.isBlank(lines) ? 128 : Integer.parseInt(lines); + reversedOrder = "y".equals(reverse); + + // TODO: remove if not necessary in future? + setCharacterSet(CharacterSet.UTF_8); + } + + @Override + public Reader getReader() throws IOException { + StringWriter writer = new StringWriter((int)fileRepresentation.getSize()+100); + write(writer); + return new StringReader(writer.toString()); + } + + /** + * Actually read the requested lines, and reverses if appropriate. + * + * If at file start, refuses to show fewer lines than are possible + * ('bounces' against start). + * + * @throws IOException + */ + protected void loadLines() throws IOException { + this.file = fileRepresentation.getFile(); + this.lines = new LinkedList(); + this.range = FileUtils.pagedLines(file, position, lineCount, lines, 128); + // bounce against the front of the file: don't show runt (fewer + // lines than requested) unless absolutely necessary) + if(lines.size()(); + this.range = FileUtils.pagedLines(file, 0, Math.abs(lineCount), lines, 128); + } + if(reversedOrder) { + Collections.reverse(lines); + } + } + + /** + * Write the paged HTML. + * + * @see org.restlet.resource.Representation#write(java.io.Writer) + */ + @Override + public void write(Writer writer) throws IOException { + loadLines(); + + PrintWriter pw = new PrintWriter(writer); + pw.println(file); + emitControls(pw); + + pw.println("
      ");
+        emitBumper(pw, true);
+        for(String line : lines) {
+            StringEscapeUtils.escapeHtml(pw,line); 
+            pw.println();
+        }
+        emitBumper(pw, false);
+        pw.println("
      "); + + emitControls(pw); + pw.close(); + } + + /** + * Emit a "start" or "EOF" bumper as appropriate to prominently + * indicate if page borders start- or end- of-file. + * + * @param pw PrintWriter + * @param atTop boolean, true if at top of page + */ + protected void emitBumper(PrintWriter pw, boolean atTop) { + if((!reversedOrder ^ atTop)&&(range.getMaximumLong()==file.length())) { + pw.println("«EOF»"); + return; + } + if((reversedOrder ^ atTop)&&(range.getMinimumLong()==0)) { + pw.println("«START»"); + } + } + + /** + * Emit the navigational controls. + * + * TODO: ugh! templatize, reduce duplication as possible + * @param pw PrintWriter + */ + protected void emitControls(PrintWriter pw) { + pw.println(""); + + if(reversedOrder) { + pw.print(""); + + pw.println(""); + + pw.println(""); + + pw.println("
      "); + pw.print("« end"); + pw.print("‹ later"); + pw.println("bytes " + +range.getMaximumLong() + +"-"+range.getMinimumLong() + +"/"+file.length() + +" " + +(int)(100*(range.getMaximumLong()/(float)file.length())) + +"%"); + pw.print("earlier ›"); + pw.print("start »"); + pw.println(""); + pw.println("normal"); + pw.println("| reversed"); + } else { + pw.print(""); + pw.print("« start"); + pw.print("‹ earlier"); + pw.println("bytes " + +range.getMinimumLong() + +"-"+range.getMaximumLong() + +"/"+file.length() + +" " + +(int)(100*(range.getMaximumLong()/(float)file.length())) + +"%"); + pw.print("later ›"); + pw.print("end »"); + pw.println("normal"); + pw.println("| reversed"); + } + + pw.print(" + "); + pw.println(lines.size()); + pw.print(" -  lines
      "); + } + + /** + * Construct navigational URI for given parameters. + * + * @param pos desired position in file + * @param lines desired signed line count + * @param reverse if line ordering should be displayed in reverse + * @return String URI appropriate to navigate to desired view + */ + protected String getControlUri(long pos, int lines, boolean reverse) { + Form query = new Form(); + query.add("format","paged"); + if(pos!=0) { + query.add("pos", Long.toString(pos)); + } + if(lines!=128) { + if(Math.abs(lines)<1) { + lines = 1; + } + query.add("lines",Integer.toString(lines)); + } + if(reverse) { + query.add("reverse","y"); + } + Reference viewRef = dirResource.getRequest().getOriginalRef().clone(); + viewRef.setQuery(query.getQueryString()); + + return viewRef.toString(); + } +} diff --git a/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java b/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java new file mode 100644 index 00000000..68ad34bf --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java @@ -0,0 +1,71 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.crawler.restlet; + +import java.util.Collection; +import java.util.logging.Logger; + +import org.restlet.Context; +import org.restlet.Guard; +import org.restlet.data.ChallengeScheme; +import org.restlet.data.Request; + +/** + * Guard that slows and logs failed authentication attempts, to make + * brute-force guessing attacks less feasible. + * + * @contributor gojomo + */ +public class RateLimitGuard extends Guard { + private static final int MIN_MS_BETWEEN_ATTEMPTS = 6000; + + private static final Logger logger = Logger.getLogger(RateLimitGuard.class.getName()); + + long lastFailureTime = 0; + + public RateLimitGuard(Context context, ChallengeScheme scheme, String realm) throws IllegalArgumentException { + super(context, scheme, realm); + } + + public RateLimitGuard(Context context, String realm, Collection baseUris, String serverKey) { + super(context, realm, baseUris, serverKey); + } + + @Override + public synchronized int authenticate(Request request) { + int retVal = super.authenticate(request); + if(retVal == AUTHENTICATION_INVALID) { + logger.warning("authentication failure "+request); + // wait until at least LAG has passed from last failure + // holding object lock the whole time, so no other checks + // can happen in parallel + long now = System.currentTimeMillis(); + long sleepMs = (lastFailureTime+MIN_MS_BETWEEN_ATTEMPTS)-now; + if(sleepMs>0) { + try { + Thread.sleep(sleepMs); + } catch (InterruptedException e) { + // ignore + } + } + lastFailureTime = now + sleepMs; + } + return retVal; + } +} diff --git a/engine/src/main/java/org/archive/crawler/spring/SheetForDecideRuled.java b/engine/src/main/java/org/archive/crawler/spring/SheetForDecideRuled.java new file mode 100644 index 00000000..5f747665 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/spring/SheetForDecideRuled.java @@ -0,0 +1,55 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.spring; + +import org.archive.modules.deciderules.DecideRule; +import org.archive.spring.Sheet; +import org.springframework.beans.factory.annotation.Required; +import org.springframework.core.Ordered; + +/** + * Sheet applied on the basis of DecideRules. + * + * @contributor gojomo + */ +public class SheetForDecideRuled extends Sheet implements Ordered, Comparable { + DecideRule rules; + int order = 0; + + public DecideRule getRules() { + return rules; + } + @Required + public void setRules(DecideRule rules) { + this.rules = rules; + } + + public int getOrder() { + return order; + } + public void setOrder(int order) { + this.order = order; + } + + // compare on the basis of Ordered value + public int compareTo(SheetForDecideRuled o) { + return order - ((Ordered)o).getOrder(); + } +} diff --git a/engine/src/main/java/org/archive/crawler/spring/SheetForSurtPrefixes.java b/engine/src/main/java/org/archive/crawler/spring/SheetForSurtPrefixes.java new file mode 100644 index 00000000..7a6f2f68 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/spring/SheetForSurtPrefixes.java @@ -0,0 +1,42 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.spring; + +import java.util.List; + +import org.archive.spring.Sheet; +import org.springframework.beans.factory.annotation.Required; + +/** + * Sheet applied on the basis of a matching SURT prefix. + * + * @contributor gojomo + */ +public class SheetForSurtPrefixes extends Sheet { + List surtPrefixes; + + public List getSurtPrefixes() { + return surtPrefixes; + } + @Required + public void setSurtPrefixes(List surtPrefixes) { + this.surtPrefixes = surtPrefixes; + } +} diff --git a/engine/src/main/java/org/archive/crawler/spring/SheetOverlaysManager.java b/engine/src/main/java/org/archive/crawler/spring/SheetOverlaysManager.java new file mode 100644 index 00000000..4cf12277 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/spring/SheetOverlaysManager.java @@ -0,0 +1,162 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.spring; + +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import org.apache.commons.collections.ListUtils; +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.spring.OverlayMapsSource; +import org.archive.spring.Sheet; +import org.archive.util.PrefixFinder; +import org.archive.util.SurtPrefixSet; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.BeanFactory; +import org.springframework.beans.factory.BeanFactoryAware; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.context.event.ContextRefreshedEvent; + +/** + * Manager which marks-up CrawlURIs with the names of all applicable + * Sheets, and returns overlay maps by name. + * + * @contributor gojomo + */ +@SuppressWarnings("unchecked") +public class SheetOverlaysManager implements +BeanFactoryAware, InitializingBean, OverlayMapsSource, ApplicationListener { + + BeanFactory beanFactory; + /** all Sheets applied by SURT-prefix */ + List surtSheets = ListUtils.EMPTY_LIST; + /** all Sheets applied by DecideRule evaluation */ + List ruleSheets = ListUtils.EMPTY_LIST; + TreeMap> sheetNamesBySurt = new TreeMap>(); + + public void setBeanFactory(BeanFactory beanFactory) throws BeansException { + this.beanFactory = beanFactory; + } + + /** + * Collect all rule-based Sheets. Typically autowired from the set + * of all SheetForDecideRuled instances. + * @param ruleSheets + */ + @SuppressWarnings("unchecked") + @Autowired(required=false) + public void setRuleSheets(List ruleSheets) { + this.ruleSheets = ruleSheets; + // always keep sorted by order + Collections.sort(this.ruleSheets); + } + + /** + * Collect all SURT-based Sheets. Typically autowired from the set + * of all SheetForSurtPrefixes instances. + * @param surtSheets + */ + @Autowired(required=false) + public void setSurtSheets(List surtSheets) { + this.surtSheets = surtSheets; + } + + /** + * After all Sheets collected, build the mapping from SURT prefixes + * to individual Sheet names. + * @see org.springframework.beans.factory.InitializingBean#afterPropertiesSet() + */ + public void afterPropertiesSet() throws Exception { + for(SheetForSurtPrefixes s : surtSheets) { + for(String prefix : s.getSurtPrefixes()) { + List sheetNames = sheetNamesBySurt.get(prefix); + if(sheetNames == null) { + sheetNames = new LinkedList(); + } + sheetNames.add(s.getBeanName()); + sheetNamesBySurt.put(prefix, sheetNames); + } + } + } + + /** + * Apply the proper overlays (by Sheet beanName) to the given CrawlURI. + * + * TODO: add guard against redundant application more than once? + * TODO: add mechanism for reapplying overlays after settings change? + * @param curi + */ + public void applyOverlays(CrawlURI curi) { + // apply SURT-based overlays + String effectiveSurt = SurtPrefixSet.getCandidateSurt(curi.getUURI()); + @SuppressWarnings("unused") + List foundPrefixes = PrefixFinder.findKeys(sheetNamesBySurt, effectiveSurt); + for(String prefix : foundPrefixes) { + for(String name : sheetNamesBySurt.get(prefix)) { + curi.getOverlayNames().push(name); + } + } + // apply deciderule-based overlays + for(SheetForDecideRuled sheet : ruleSheets) { + if(sheet.getRules().accepts(curi)) { + curi.getOverlayNames().addFirst(sheet.getBeanName()); + } + } + // even if no overlays set, let creation of empty list signal + // step has occurred -- helps ensure overlays added once-only + curi.getOverlayNames(); + } + + /** + * Retrieve the named overlay Map. + * + * @see org.archive.spring.OverlayMapsSource#getOverlayMap(java.lang.String) + */ + public Map getOverlayMap(String name) { + Sheet sheet = (Sheet) beanFactory.getBean(name, Sheet.class); + return sheet.getMap(); + } + + /** + * Ensure all sheets are 'primed' after the entire ApplicationContext + * is assembled. This ensures target HasKeyedProperties beans know + * any long paths by which their properties are addressed, and + * handles (by either PropertyEditor-conversion or a fast-failure) + * any type-mismatches between overlay values and their target + * properties. + * @see org.springframework.context.ApplicationListener#onApplicationEvent(org.springframework.context.ApplicationEvent) + */ + public void onApplicationEvent(ApplicationEvent event) { + if(event instanceof ContextRefreshedEvent) { + for(Sheet s: surtSheets) { + s.prime(); + } + for(Sheet s: ruleSheets) { + s.prime(); + } + } + } +} \ No newline at end of file diff --git a/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java b/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java new file mode 100644 index 00000000..17384b9b --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java @@ -0,0 +1,386 @@ +/* BdbUriUniqFilter +* +* $Id$ +* +* Created on September 17, 2004 +* +* Copyright (C) 2004 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.crawler.util; + +import java.io.File; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.archive.bdb.BdbModule; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.Lifecycle; + +import st.ata.util.FPGenerator; + +import com.sleepycat.bind.tuple.LongBinding; +import com.sleepycat.je.Database; +import com.sleepycat.je.DatabaseEntry; +import com.sleepycat.je.DatabaseException; +import com.sleepycat.je.DatabaseNotFoundException; +import com.sleepycat.je.Environment; +import com.sleepycat.je.EnvironmentConfig; +import com.sleepycat.je.OperationStatus; + + +/** + * A BDB implementation of an AlreadySeen list. + * + * This implementation performs adequately without blowing out + * the heap. See + * AlreadySeen. + * + *

      Makes keys that have URIs from same server close to each other. Mercator + * and 2.3.5 'Elminating Already-Visited URLs' in 'Mining the Web' by Soumen + * Chakrabarti talk of a two-level key with the first 24 bits a hash of the + * host plus port and with the last 40 as a hash of the path. Testing + * showed adoption of such a scheme halving lookup times (Tutilhis implementation + * actually concatenates scheme + host in first 24 bits and path + query in + * trailing 40 bits). + * + * @author stack + * @version $Date$, $Revision$ + */ +public class BdbUriUniqFilter extends SetBasedUriUniqFilter +implements Lifecycle, Serializable { + private static final long serialVersionUID = -8099357538178524011L; + + private static Logger logger = + Logger.getLogger(BdbUriUniqFilter.class.getName()); + + protected boolean createdEnvironment = false; + protected long lastCacheMiss = 0; + protected long lastCacheMissDiff = 0; + protected transient Database alreadySeen = null; + protected transient DatabaseEntry value = null; + static protected DatabaseEntry ZERO_LENGTH_ENTRY = + new DatabaseEntry(new byte[0]); + private static final String DB_NAME = "alreadySeenUrl"; + protected long count = 0; + private long aggregatedLookupTime = 0; + + private static final String COLON_SLASH_SLASH = "://"; + + protected BdbModule bdb; + @Autowired + public void setBdbModule(BdbModule bdb) { + this.bdb = bdb; + } + + public BdbUriUniqFilter() { + } + + boolean isRunning = false; + public void start() { + if(isRunning()) { + return; + } + try { + BdbModule.BdbConfig config = getDatabaseConfig(); + config.setAllowCreate(true); + initialize(bdb.openDatabase(DB_NAME, config, false)); + } catch (DatabaseException e) { + throw new IllegalStateException(e); + } + isRunning = true; + } + + public boolean isRunning() { + return isRunning; + } + + public void stop() { + if(!isRunning()) { + return; + } + close(); + isRunning = false; + } + + /** + * Constructor. + * + * Only used for testing; usually no-arg constructor is used, and + * environment provided by injected BdbModule. + * + * @param bdbEnv The directory that holds the bdb environment. Will + * make a database under here if doesn't already exit. Otherwise + * reopens any existing dbs. + * @throws IOException + */ + public BdbUriUniqFilter(File bdbEnv) + throws IOException { + this(bdbEnv, -1); + } + + /** + * Constructor. + * + * Only used for testing; usually no-arg constructor is used, and + * environment provided by injected BdbModule. + * + * @param bdbEnv The directory that holds the bdb environment. Will + * make a database under here if doesn't already exit. Otherwise + * reopens any existing dbs. + * @param cacheSizePercentage Percentage of JVM bdb allocates as + * its cache. Pass -1 to get default cache size. + * @throws IOException + */ + public BdbUriUniqFilter(File bdbEnv, final int cacheSizePercentage) + throws IOException { + super(); + if (!bdbEnv.exists()) { + bdbEnv.mkdirs(); + } + EnvironmentConfig envConfig = new EnvironmentConfig(); + envConfig.setAllowCreate(true); + if (cacheSizePercentage > 0 && cacheSizePercentage < 100) { + envConfig.setCachePercent(cacheSizePercentage); + } + try { + createdEnvironment = true; + Environment env = new Environment(bdbEnv, envConfig); + BdbModule.BdbConfig config = getDatabaseConfig(); + config.setAllowCreate(true); + try { + env.truncateDatabase(null, DB_NAME, false); + } catch (DatabaseNotFoundException e) { + // ignored + } + Database db = env.openDatabase(null, DB_NAME, config.toDatabaseConfig()); + initialize(db); + } catch (DatabaseException e) { + IOException io = new IOException(); + io.initCause(e); + throw io; + } + } + + + /** + * Method shared by constructors. + * @param env Environment to use. + * @throws DatabaseException + */ + protected void initialize(Database db) throws DatabaseException { + open(db); + } + + /** + * @return DatabaseConfig to use + */ + protected BdbModule.BdbConfig getDatabaseConfig() { + BdbModule.BdbConfig dbConfig = new BdbModule.BdbConfig(); + return dbConfig; + } + + /** + * Call after deserializing an instance of this class. Will open the + * already seen in passed environment. + * @param env DB Environment to use. + * @throws DatabaseException + */ + public void reopen(Database db) + throws DatabaseException { + open(db); + } + + protected void open(final Database db) + throws DatabaseException { + this.alreadySeen = db; + this.value = new DatabaseEntry("".getBytes()); + } + + public synchronized void close() { + if (logger.isLoggable(Level.INFO)) { + logger.info("Count of alreadyseen on close " + + Long.toString(count)); + } + Environment env = null; + if (this.alreadySeen != null) { + try { + env = this.alreadySeen.getEnvironment(); + alreadySeen.sync(); + } catch (DatabaseException e) { + logger.severe(e.getMessage()); + } + } + if (env != null) { + try { + // This sync flushes whats in RAM. Its expensive operation. + // Without, data can be lost. Not for transactional operation. + env.sync(); + } catch (DatabaseException e) { + logger.severe(e.getMessage()); + } + } + + if (createdEnvironment) { + // Only manually close database if it were created via a + // constructor, and not via a BdbModule. Databases created by a + // BdbModule will be closed by that BdbModule. + if (this.alreadySeen != null) { + try { + alreadySeen.close(); + } catch (DatabaseException e) { + logger.severe(e.getMessage()); + } + } + if (env != null) { + try { + env.close(); + } catch (DatabaseException e) { + logger.severe(e.getMessage()); + } + } + } + } + + public synchronized long getCacheMisses() throws DatabaseException { + long cacheMiss = this.alreadySeen.getEnvironment(). + getStats(null).getNCacheMiss(); + this.lastCacheMissDiff = cacheMiss - this.lastCacheMiss; + this.lastCacheMiss = cacheMiss; + return this.lastCacheMiss; + } + + public long getLastCacheMissDiff() { + return this.lastCacheMissDiff; + } + + /** + * Create fingerprint. + * Pubic access so test code can access createKey. + * @param uri URI to fingerprint. + * @return Fingerprint of passed url. + */ + public static long createKey(CharSequence uri) { + String url = uri.toString(); + int index = url.indexOf(COLON_SLASH_SLASH); + if (index > 0) { + index = url.indexOf('/', index + COLON_SLASH_SLASH.length()); + } + CharSequence hostPlusScheme = (index == -1)? url: url.subSequence(0, index); + long tmp = FPGenerator.std24.fp(hostPlusScheme); + return tmp | (FPGenerator.std40.fp(url) >>> 24); + } + + + + protected boolean setAdd(CharSequence uri) { + DatabaseEntry key = new DatabaseEntry(); + LongBinding.longToEntry(createKey(uri), key); + long started = 0; + + OperationStatus status = null; + try { + if (logger.isLoggable(Level.INFO)) { + started = System.currentTimeMillis(); + } + status = alreadySeen.putNoOverwrite(null, key, ZERO_LENGTH_ENTRY); + if (logger.isLoggable(Level.INFO)) { + aggregatedLookupTime += + (System.currentTimeMillis() - started); + } + } catch (DatabaseException e) { + logger.severe(e.getMessage()); + } + if (status == OperationStatus.SUCCESS) { + count++; + if (logger.isLoggable(Level.INFO)) { + final int logAt = 10000; + if (count > 0 && ((count % logAt) == 0)) { + logger.info("Average lookup " + + (aggregatedLookupTime / logAt) + "ms."); + aggregatedLookupTime = 0; + } + } + } + if(status == OperationStatus.KEYEXIST) { + return false; // not added + } else { + return true; + } + } + + protected long setCount() { + return count; + } + + protected boolean setRemove(CharSequence uri) { + DatabaseEntry key = new DatabaseEntry(); + LongBinding.longToEntry(createKey(uri), key); + OperationStatus status = null; + try { + status = alreadySeen.delete(null, key); + } catch (DatabaseException e) { + logger.severe(e.getMessage()); + } + if (status == OperationStatus.SUCCESS) { + count--; + return true; // removed + } else { + return false; // not present + } + } + + public long flush() { + // We always write but this might be place to do the sync + // when checkpointing? TODO. + return 0; + } + + private void writeObject(ObjectOutputStream output) throws IOException { + // sync deferred-write database + try { + alreadySeen.sync(); + } catch (DatabaseException e) { + // TODO Auto-generated catch block + throw new RuntimeException(e); + } + output.defaultWriteObject(); + } + + private void readObject(ObjectInputStream input) + throws IOException, ClassNotFoundException { + input.defaultReadObject(); + + try { + BdbModule.BdbConfig config = getDatabaseConfig(); + config.setAllowCreate(false); + reopen(bdb.getDatabase(DB_NAME)); + } catch (DatabaseException e) { + IOException io = new IOException(); + io.initCause(e); + throw io; + } + } + + + +} \ No newline at end of file diff --git a/engine/src/main/java/org/archive/crawler/util/BenchmarkUriUniqFilters.java b/engine/src/main/java/org/archive/crawler/util/BenchmarkUriUniqFilters.java new file mode 100644 index 00000000..f3be6d74 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/util/BenchmarkUriUniqFilters.java @@ -0,0 +1,129 @@ +/* BenchmarkUriUniqFilters + * + * $Id$ + * + * Created on Jun 22, 2005. + * + * Copyright (C) 2005 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.crawler.util; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; + +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.crawler.datamodel.UriUniqFilter; +import org.archive.util.fingerprint.MemLongFPSet; + + +/** + * BenchmarkUriUniqFilters + * + * @author gojomo + */ +public class BenchmarkUriUniqFilters implements UriUniqFilter.CrawlUriReceiver { +// private Logger LOGGER = +// Logger.getLogger(BenchmarkUriUniqFilters.class.getName()); + + private BufferedWriter out; // optional to dump uniq items + String current; // current line/URI being checked + + /** + * Test the UriUniqFilter implementation (MemUriUniqFilter, + * BloomUriUniqFilter, or BdbUriUniqFilter) named in first + * argument against the file of one-per-line URIs named + * in the second argument. + * + * @param args from cmd-line + * @throws IOException + */ + public static void main(String[] args) throws IOException { + (new BenchmarkUriUniqFilters()).instanceMain(args); + } + + public void instanceMain(String[] args) throws IOException { + String testClass = args[0]; + String inputFilename = args[1]; + long start = System.currentTimeMillis(); + UriUniqFilter uniq = createUriUniqFilter(testClass); + long created = System.currentTimeMillis(); + BufferedReader br = new BufferedReader(new FileReader(inputFilename)); + if(args.length>2) { + String outputFilename = args[2]; + out = new BufferedWriter(new FileWriter(outputFilename)); + } + int added = 0; + while((current=br.readLine())!=null) { + added++; + uniq.add(current,null); + } + uniq.close(); + long finished = System.currentTimeMillis(); + if(out!=null) { + out.close(); + } + System.out.println(added+" adds"); + System.out.println(uniq.count()+" retained"); + System.out.println((created-start)+"ms to setup UUF"); + System.out.println((finished-created)+"ms to perform all adds"); + } + + private UriUniqFilter createUriUniqFilter(String testClass) throws IOException { + UriUniqFilter uniq = null; + if(BdbUriUniqFilter.class.getName().endsWith(testClass)) {; + // BDB setup + File tmpDir = File.createTempFile("uuf","benchmark"); + tmpDir.delete(); + tmpDir.mkdir(); + uniq = new BdbUriUniqFilter(tmpDir, 50); + } else if(BloomUriUniqFilter.class.getName().endsWith(testClass)) { + // bloom setup + uniq = new BloomUriUniqFilter(); + } else if(MemUriUniqFilter.class.getName().endsWith(testClass)) { + // mem hashset + uniq = new MemUriUniqFilter(); + } else if (FPUriUniqFilter.class.getName().endsWith(testClass)) { + // mem fp set (open-addressing) setup + uniq = new FPUriUniqFilter(new MemLongFPSet(21,0.75f)); + } + uniq.setDestination(this); + return uniq; + } + + /* (non-Javadoc) + * @see org.archive.crawler.datamodel.UriUniqFilter.HasUriReceiver#receive(org.archive.crawler.datamodel.CrawlURI) + */ + public void receive(CrawlURI item) { + if(out!=null) { + try { + // we assume all tested filters are immediate passthrough so + // we can use 'current'; a buffering filter would change this + // assumption + out.write(current); + out.write("\n"); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } +} \ No newline at end of file diff --git a/engine/src/main/java/org/archive/crawler/util/BloomUriUniqFilter.java b/engine/src/main/java/org/archive/crawler/util/BloomUriUniqFilter.java new file mode 100644 index 00000000..268f950a --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/util/BloomUriUniqFilter.java @@ -0,0 +1,151 @@ +/* BloomUriUniqFilter +* +* $Id$ +* +* Created on June 21, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.crawler.util; + +import java.io.Serializable; +import java.util.logging.Logger; + +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.util.BloomFilter; +import org.archive.util.BloomFilter32bitSplit; + + +/** + * A MG4J BloomFilter-based implementation of an AlreadySeen list. + * + * This implementation performs adequately without blowing out + * the heap through to very large numbers of URIs. See + * AlreadySeen. + * + * It is inherent to Bloom filters that as they get 'saturated', their + * false-positive rate rises. The default parameters used by this class + * attempt to maintain a 1-in-4 million (1 in 2^22) false-positive chance + * through 125 million unique inserts, which creates a filter structure + * about 495MB in size. + * + * You may use the following system properties to tune the size and + * false-positive rate of the bloom filter structure used by this class: + * + * org.archive.crawler.util.BloomUriUniqFilter.expected-size (default 125000000) + * org.archive.crawler.util.BloomUriUniqFilter.hash-count (default 22) + * + * The resulting filter will take up approximately... + * + * 1.44 * expected-size * hash-count / 8 + * + * ...bytes. + * + * The default size is very close to the maximum practical size of the + * Bloom filter implementation, BloomFilter32bitSplit, created in the + * initialize() method, due to integer arithmetic limits. + * + * If you need a larger filter, you should edit the initialize + * method to intantiate a BloomFilter64bit instead. + * + * @author gojomo + * @version $Date$, $Revision$ + */ +public class BloomUriUniqFilter extends SetBasedUriUniqFilter +implements Serializable { + private static final long serialVersionUID = 1061526253773091309L; + + private static Logger LOGGER = + Logger.getLogger(BloomUriUniqFilter.class.getName()); + + BloomFilter bloom; // package access for testing convenience + protected int expected_n; // remember bloom contruction param + + protected static final String EXPECTED_SIZE_KEY = ".expected-size"; + protected static final String HASH_COUNT_KEY = ".hash-count"; + + // these defaults create a bloom filter that is + // 1.44*125mil*22/8 ~= 495MB in size, and at full + // capacity will give a false contained indication + // 1/(2^22) ~= 1 in every 4 million probes + private static final int DEFAULT_EXPECTED_SIZE = 125000000; // 125 million + private static final int DEFAULT_HASH_COUNT = 22; // 1 in 4 million false pos + + /** + * Default constructor + */ + public BloomUriUniqFilter() { + super(); + String ns = System.getProperty(this.getClass().getName() + EXPECTED_SIZE_KEY); + int n = (ns == null) ? DEFAULT_EXPECTED_SIZE : Integer.parseInt(ns); + String ds = System.getProperty(this.getClass().getName() + HASH_COUNT_KEY); + int d = (ds == null) ? DEFAULT_HASH_COUNT : Integer.parseInt(ds); + initialize(n,d); + } + + /** + * Constructor. + * + * @param n the expected number of elements. + * @param d the number of hash functions; if the filter adds not more + * than n elements, false positives will happen with + * probability 2-d. + */ + public BloomUriUniqFilter( final int n, final int d ) { + super(); + initialize(n, d); + } + + /** + * Initializer shared by constructors. + * + * @param n the expected number of elements. + * @param d the number of hash functions; if the filter adds not more + * than n elements, false positives will happen with + * probability 2-d. + */ + protected void initialize(final int n, final int d) { + this.expected_n = n; + bloom = new BloomFilter32bitSplit(n,d); + } + + public void forget(String canonical, CrawlURI item) { + // TODO? could use in-memory exception list of currently-forgotten items + LOGGER.severe("forget(\""+canonical+"\",CrawlURI) not supported"); + } + + + protected boolean setAdd(CharSequence uri) { + boolean added = bloom.add(uri); + // warn if bloom has reached its expected size (and its false-pos + // rate will now exceed the theoretical/designed level) + if( added && (count() == expected_n)) { + LOGGER.warning("Bloom has reached expected limit "+expected_n); + } + return added; + } + + protected long setCount() { + return bloom.size(); + } + + protected boolean setRemove(CharSequence uri) { + throw new UnsupportedOperationException(); + } +} diff --git a/engine/src/main/java/org/archive/crawler/util/CheckpointUtils.java b/engine/src/main/java/org/archive/crawler/util/CheckpointUtils.java new file mode 100644 index 00000000..2d91ec60 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/util/CheckpointUtils.java @@ -0,0 +1,126 @@ +/* CheckpointUtils + * + * $Id$ + * + * Created on December 16, 2005. + * + * Copyright (C) 2005 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.crawler.util; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; + +/** + * Utilities useful checkpointing. + * @author stack + * @version $Date$ $Revision$ + */ +public class CheckpointUtils { + public static final String SERIALIZED_CLASS_SUFFIX = ".serialized"; + + public static File getBdbSubDirectory(File checkpointDir) { + return new File(checkpointDir, "bdbje-logs"); + } + + public static File getClassCheckpointFile(File checkpointDir, + final String suffix, Class c) { + return new File(checkpointDir, getClassCheckpointFilename(c, suffix)); + } + + public static File getClassCheckpointFile(File checkpointDir, Class c) { + return new File(checkpointDir, getClassCheckpointFilename(c, null)); + } + + public static String getClassCheckpointFilename(final Class c) { + return getClassCheckpointFilename(c, null); + } + + public static String getClassCheckpointFilename(final Class c, + final String suffix) { + return c.getName() + ((suffix == null)? "": "." + suffix) + + SERIALIZED_CLASS_SUFFIX; + } + + /** + * Utility function to serialize an object to a file in current checkpoint + * dir. Facilities + * to store related files alongside the serialized object in a directory + * named with a .auxillary suffix. + * + * @param o Object to serialize. + * @param dir Directory to serialize into. + * @throws IOException + */ + public static void writeObjectToFile(final Object o, final File dir) + throws IOException { + writeObjectToFile(o, null, dir); + } + + public static void writeObjectToFile(final Object o, final String suffix, + final File dir) + throws IOException { + dir.mkdirs(); + ObjectOutputStream out = new ObjectOutputStream( + new FileOutputStream(getClassCheckpointFile(dir, suffix, + o.getClass()))); + try { + out.writeObject(o); + } finally { + out.close(); + } + } + + public static T readObjectFromFile(final Class c, final File dir) + throws FileNotFoundException, IOException, ClassNotFoundException { + return readObjectFromFile(c, null, dir); + } + + public static T readObjectFromFile(final Class c, final String suffix, + final File dir) + throws FileNotFoundException, IOException, ClassNotFoundException { + ObjectInputStream in = new ObjectInputStream( + new FileInputStream(getClassCheckpointFile(dir, suffix, c))); + T o = null; + try { + o = c.cast(in.readObject()); + } finally { + in.close(); + } + return o; + } + + /** + * @return Instance of filename filter that will let through files ending + * in '.jdb' (i.e. bdb je log files). + */ + public static FilenameFilter getJeLogsFilter() { + return new FilenameFilter() { + public boolean accept(File dir, String name) { + return name != null && name.toLowerCase().endsWith(".jdb"); + } + }; + } +} diff --git a/engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java b/engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java new file mode 100644 index 00000000..fe21be8d --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java @@ -0,0 +1,73 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.util; + +import org.apache.commons.httpclient.HttpStatus; +import org.archive.crawler.datamodel.CoreAttributeConstants; +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.deciderules.recrawl.IdenticalDigestDecideRule; +import org.archive.util.ArchiveUtils; +import org.archive.util.Histotable; + +public class CrawledBytesHistotable extends Histotable +implements CoreAttributeConstants { + private static final long serialVersionUID = 7923431123239026213L; + + public static final String NOTMODIFIED = "not-modified"; + public static final String DUPLICATE = "dup-by-hash"; + public static final String NOVEL = "novel"; + + + public CrawledBytesHistotable() { + super(); + } + + public void accumulate(CrawlURI curi) { + if(curi.getFetchStatus()==HttpStatus.SC_NOT_MODIFIED) { + tally(NOTMODIFIED, curi.getContentSize()); + } else if (IdenticalDigestDecideRule.hasIdenticalDigest(curi)) { + tally(DUPLICATE,curi.getContentSize()); + } else { + tally(NOVEL,curi.getContentSize()); + } + } + + public String summary() { + StringBuilder sb = new StringBuilder(); + sb.append(ArchiveUtils.formatBytesForDisplay(getTotal())); + sb.append(" crawled ("); + sb.append(ArchiveUtils.formatBytesForDisplay(get(NOVEL))); + sb.append(" novel"); + if(get(DUPLICATE)!=null) { + sb.append(", "); + sb.append(ArchiveUtils.formatBytesForDisplay(get(DUPLICATE))); + sb.append(" "); + sb.append(DUPLICATE); + } + if(get(NOTMODIFIED)!=null) { + sb.append(", "); + sb.append(ArchiveUtils.formatBytesForDisplay(get(NOTMODIFIED))); + sb.append(" "); + sb.append(NOTMODIFIED); + } + sb.append(")"); + return sb.toString(); + } +} diff --git a/engine/src/main/java/org/archive/crawler/util/DiskFPMergeUriUniqFilter.java b/engine/src/main/java/org/archive/crawler/util/DiskFPMergeUriUniqFilter.java new file mode 100644 index 00000000..cea2d2f1 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/util/DiskFPMergeUriUniqFilter.java @@ -0,0 +1,212 @@ +/* DiskFPMergeUriUniqFilter +* +* $Id$ +* +* Created on Dec 14, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.crawler.util; + +import it.unimi.dsi.fastutil.longs.LongIterators; +import it.unimi.dsi.fastutil.longs.LongIterator; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.NoSuchElementException; + +import org.archive.util.ArchiveUtils; + +/** + * Crude FPMergeUriUniqFilter using a disk data file of raw longs as the + * overall FP record. + * + * @author gojomo + */ +public class DiskFPMergeUriUniqFilter extends FPMergeUriUniqFilter { + long count = 0; + File scratchDir; + File currentFps; + File newFpsFile; + DataOutputStream newFps; + long newCount; + DataInputStream oldFps; + + public DiskFPMergeUriUniqFilter(File scratchDir) { + super(); + this.scratchDir = scratchDir; + // TODO: Use two scratch locations, to allow IO to be split + // over separate disks + } + + /* (non-Javadoc) + * @see org.archive.crawler.util.FPMergeUriUniqFilter#beginFpMerge() + */ + protected LongIterator beginFpMerge() { + newFpsFile = new File(scratchDir,ArchiveUtils.get17DigitDate()+".fp"); + if(newFpsFile.exists()) { + throw new RuntimeException(newFpsFile+" exists"); + } + try { + newFps = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(newFpsFile))); + } catch (FileNotFoundException e) { + throw new RuntimeException(e); + } + newCount = 0; + if(currentFps==null) { + return LongIterators.EMPTY_ITERATOR; + } + try { + oldFps = new DataInputStream(new BufferedInputStream(new FileInputStream(currentFps))); + } catch (FileNotFoundException e1) { + throw new RuntimeException(e1); + } + return new DataFileLongIterator(oldFps); + } + + /* (non-Javadoc) + * @see org.archive.crawler.util.FPMergeUriUniqFilter#addNewFp(long) + */ + protected void addNewFp(long fp) { + try { + newFps.writeLong(fp); + newCount++; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /* (non-Javadoc) + * @see org.archive.crawler.util.FPMergeUriUniqFilter#finishFpMerge() + */ + protected void finishFpMerge() { + try { + newFps.close(); + File oldFpsFile = currentFps; + currentFps = newFpsFile; + if(oldFps!=null) { + oldFps.close(); + } + if(oldFpsFile!=null) { + oldFpsFile.delete(); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + count = newCount; + } + + /* (non-Javadoc) + * @see org.archive.crawler.datamodel.UriUniqFilter#count() + */ + public long count() { + return count; + } + + public class DataFileLongIterator implements LongIterator { + DataInputStream in; + long next; + boolean nextIsValid = false; + + /** + * Construct a long iterator reading from the given + * stream. + * + * @param disStream DataInputStream from which to read longs + */ + public DataFileLongIterator(DataInputStream disStream) { + this.in = disStream; + } + + /** + * Test whether any items remain; loads next item into + * holding 'next' field. + * + * @see java.util.Iterator#hasNext() + */ + public boolean hasNext() { + return nextIsValid ? true: lookahead(); + } + + /** + * Check if there's a next by trying to read it. + * + * @return true if 'next' field is filled with a valid next, false otherwise + */ + protected boolean lookahead() { + try { + next = in.readLong(); + } catch (IOException e) { + return false; + } + nextIsValid = true; + return true; + } + + /** + * Return the next item. + * + * @see java.util.Iterator#next() + */ + public Long next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + // 'next' is guaranteed set by a hasNext() which returned true + Long returnObj = new Long(this.next); + this.nextIsValid = false; + return returnObj; + } + + /* (non-Javadoc) + * @see java.util.Iterator#remove() + */ + public void remove() { + throw new UnsupportedOperationException(); + } + + + /* (non-Javadoc) + * @see it.unimi.dsi.fastutil.longs.LongIterator#nextLong() + */ + public long nextLong() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + // 'next' is guaranteed non-null by a hasNext() which returned true + this.nextIsValid = false; // after this return, 'next' needs refresh + return this.next; + } + + /* (non-Javadoc) + * @see it.unimi.dsi.fastutil.longs.LongIterator#skip(int) + */ + public int skip(int arg0) { + return 0; + } + } + +} diff --git a/engine/src/main/java/org/archive/crawler/util/FPMergeUriUniqFilter.java b/engine/src/main/java/org/archive/crawler/util/FPMergeUriUniqFilter.java new file mode 100644 index 00000000..5782469c --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/util/FPMergeUriUniqFilter.java @@ -0,0 +1,364 @@ +/* UriUniqFilterImpl +* +* $Id$ +* +* Created on Sep 29, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.crawler.util; + +import it.unimi.dsi.fastutil.longs.LongIterator; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.PrintWriter; +import java.util.Iterator; +import java.util.TreeSet; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.crawler.datamodel.UriUniqFilter; +import org.archive.util.fingerprint.ArrayLongFPCache; + +import st.ata.util.FPGenerator; + +/** + * UriUniqFilter based on merging FP arrays (in memory or from disk). + * + * Inspired by the approach in Najork and Heydon, "High-Performance + * Web Crawling" (2001), section 3.2, "Efficient Duplicate URL + * Eliminators". + * + * @author gojomo + */ +public abstract class FPMergeUriUniqFilter implements UriUniqFilter { + /** + * Represents a long fingerprint and (possibly) its corresponding + * CrawlURI, awaiting the next merge in a 'pending' state. + */ + public class PendingItem implements Comparable { + long fp; + CrawlURI caUri; + public PendingItem(long fp, CrawlURI value) { + this.fp = fp; + this.caUri = value; + } + public int compareTo(PendingItem vs) { + return (fp < vs.fp) ? -1 : ( (fp == vs.fp) ? 0 : 1); + } + } + + private static Logger LOGGER = + Logger.getLogger(FPMergeUriUniqFilter.class.getName()); + + protected CrawlUriReceiver receiver; + protected PrintWriter profileLog; + + // statistics + protected long quickDuplicateCount = 0; + protected long quickDupAtLast = 0; + protected long pendDuplicateCount = 0; + protected long pendDupAtLast = 0; + protected long mergeDuplicateCount = 0; + protected long mergeDupAtLast = 0; + + /** items awaiting merge + * TODO: consider only sorting just pre-merge + * TODO: consider using a fastutil long->Object class + * TODO: consider actually writing items to disk file, + * as in Najork/Heydon + */ + protected TreeSet pendingSet = new TreeSet(); + + /** size at which to force flush of pending items */ + protected int maxPending = DEFAULT_MAX_PENDING; + public static final int DEFAULT_MAX_PENDING = 10000; + // TODO: increase + + /** + * time-based throttle on flush-merge operations + */ + protected long nextFlushAllowableAfter = 0; + public static final long FLUSH_DELAY_FACTOR = 100; + + /** cache of most recently seen FPs */ + protected ArrayLongFPCache quickCache = new ArrayLongFPCache(); + // TODO: make cache most-often seen, not just most-recent + + public FPMergeUriUniqFilter() { + super(); + String profileLogFile = + System.getProperty(FPMergeUriUniqFilter.class.getName() + + ".profileLogFile"); + if (profileLogFile != null) { + setProfileLog(new File(profileLogFile)); + } + } + + public void setMaxPending(int max) { + maxPending = max; + } + + public long pending() { + return pendingSet.size(); + } + + public void setDestination(CrawlUriReceiver receiver) { + this.receiver = receiver; + } + + protected void profileLog(String key) { + if (profileLog != null) { + profileLog.println(key); + } + } + + /* (non-Javadoc) + * @see org.archive.crawler.datamodel.UriUniqFilter#add(java.lang.String, org.archive.crawler.datamodel.CrawlURI) + */ + public synchronized void add(String key, CrawlURI value) { + profileLog(key); + long fp = createFp(key); + if(! quickCheck(fp)) { + quickDuplicateCount++; + return; + } + pend(fp,value); + if (pendingSet.size()>=maxPending) { + flush(); + } + } + + /** + * Place the given FP/CrawlURI pair into the pending set, awaiting + * a merge to determine if it's actually accepted. + * + * @param fp long fingerprint + * @param value CrawlURI or null, if fp only needs merging (as when + * CrawlURI was already forced in + */ + protected void pend(long fp, CrawlURI value) { + // special case for first batch of adds + if(count()==0) { + if(pendingSet.add(new PendingItem(fp,null))==false) { + pendDuplicateCount++; // was already present + } else { + // since there's no prior list to merge, push uri along right now + if(value!=null) { + this.receiver.receive(value); + } + } + return; + } + if(pendingSet.add(new PendingItem(fp,value))==false) { + pendDuplicateCount++; // was already present + } + } + + /** + * Evaluate if quick-check cache considers fingerprint novel enough + * for further consideration. + * + * @param fp long fingerprint to check + * @return true if fp deserves consideration; false if it appears in cache + */ + private boolean quickCheck(long fp) { + return quickCache.add(fp); + } + + /** + * Create a fingerprint from the given key + * + * @param key CharSequence (URI) to fingerprint + * @return long fingerprint + */ + public static long createFp(CharSequence key) { + return FPGenerator.std64.fp(key); + } + + + /* (non-Javadoc) + * @see org.archive.crawler.datamodel.UriUniqFilter#addNow(java.lang.String, org.archive.crawler.datamodel.CrawlURI) + */ + public void addNow(String key, CrawlURI value) { + add(key, value); + flush(); + } + + /* (non-Javadoc) + * @see org.archive.crawler.datamodel.UriUniqFilter#addForce(java.lang.String, org.archive.crawler.datamodel.CrawlURI) + */ + public void addForce(String key, CrawlURI value) { + add(key,null); // dummy pend + this.receiver.receive(value); + } + + /* (non-Javadoc) + * @see org.archive.crawler.datamodel.UriUniqFilter#note(java.lang.String) + */ + public void note(String key) { + add(key,null); + } + + /* (non-Javadoc) + * @see org.archive.crawler.datamodel.UriUniqFilter#forget(java.lang.String, org.archive.crawler.datamodel.CrawlURI) + */ + public void forget(String key, CrawlURI value) { + throw new UnsupportedOperationException(); + } + + /* (non-Javadoc) + * @see org.archive.crawler.datamodel.UriUniqFilter#requestFlush() + */ + public synchronized long requestFlush() { + if(System.currentTimeMillis()>nextFlushAllowableAfter) { + return flush(); + } else { +// LOGGER.info("declining to flush: too soon after last flush"); + return -1; + } + } + + /** + * Perform a merge of all 'pending' items to the overall fingerprint list. + * If the pending item is new, and has an associated CrawlURI, pass that + * URI along to the 'receiver' (frontier) for queueing. + * + * @return number of pending items actually added + */ + public synchronized long flush() { + if(pending()==0) { + return 0; + } + long flushStartTime = System.currentTimeMillis(); + long adds = 0; + long fpOnlyAdds = 0; + Long currFp = null; + PendingItem currPend = null; + + Iterator pendIter = pendingSet.iterator(); + LongIterator fpIter = beginFpMerge(); + + currPend = (PendingItem) (pendIter.hasNext() ? pendIter.next() : null); + currFp = (Long) (fpIter.hasNext() ? fpIter.next() : null); + + while(true) { + while(currFp!=null && (currPend==null||(currFp.longValue() <= currPend.fp))) { + addNewFp(currFp.longValue()); + if(currPend!=null && currFp.longValue() == currPend.fp) { + mergeDuplicateCount++; + } + if(fpIter.hasNext()) { + currFp = (Long) fpIter.next(); + } else { + currFp = null; + break; + } + } + while(currPend!=null && (currFp==null||(currFp.longValue() > currPend.fp))) { + addNewFp(currPend.fp); + if(currPend.caUri!=null) { + adds++; + this.receiver.receive(currPend.caUri); + } else { + fpOnlyAdds++; + } + if(pendIter.hasNext()) { + currPend = (PendingItem)pendIter.next(); + } else { + currPend = null; + break; + } + } + if(currFp==null) { + // currPend must be null too, or while wouldn't have exitted + // done + break; + } + } + // maintain throttle timing + long flushDuration = System.currentTimeMillis() - flushStartTime; + nextFlushAllowableAfter = flushStartTime + (FLUSH_DELAY_FACTOR*flushDuration); + + // add/duplicate statistics + if(LOGGER.isLoggable(Level.INFO)) { + long mergeDups = (mergeDuplicateCount-mergeDupAtLast); + long pendDups = (pendDuplicateCount-pendDupAtLast); + long quickDups = (quickDuplicateCount-quickDupAtLast); + LOGGER.info("flush took "+flushDuration+"ms: " + +adds+" adds, " + +fpOnlyAdds+" fpOnlydds, " + +mergeDups+" mergeDups, " + +pendDups+" pendDups, " + +quickDups+" quickDups "); + if(adds==0 && fpOnlyAdds==0 && mergeDups == 0 && pendDups == 0 && quickDups == 0) { + LOGGER.info("that's odd"); + } + } + mergeDupAtLast = mergeDuplicateCount; + pendDupAtLast = pendDuplicateCount; + quickDupAtLast = quickDuplicateCount; + pendingSet.clear(); + finishFpMerge(); + return adds; + } + + /** + * Begin merging pending candidates with complete list. Return an + * Iterator which will return all previously-known FPs in turn. + * + * @return Iterator over all previously-known FPs + */ + abstract protected LongIterator beginFpMerge(); + + + /** + * Add an FP (which may be an old or new FP) to the new complete + * list. Should only be called after beginFpMerge() and before + * finishFpMerge(). + * + * @param fp the FP to add + */ + abstract protected void addNewFp(long fp); + + /** + * Complete the merge of candidate and previously-known FPs (closing + * files/iterators as appropriate). + */ + abstract protected void finishFpMerge(); + + public void close() { + if (profileLog != null) { + profileLog.close(); + } + } + + public void setProfileLog(File logfile) { + try { + profileLog = new PrintWriter(new BufferedOutputStream( + new FileOutputStream(logfile))); + } catch (FileNotFoundException e) { + throw new RuntimeException(e); + } + } +} diff --git a/engine/src/main/java/org/archive/crawler/util/FPUriUniqFilter.java b/engine/src/main/java/org/archive/crawler/util/FPUriUniqFilter.java new file mode 100644 index 00000000..d572f16a --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/util/FPUriUniqFilter.java @@ -0,0 +1,83 @@ +/* Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * MemFPUURISet.java + * Created on Oct 1, 2003 + * + * $Header$ + */ +package org.archive.crawler.util; + +import java.io.Serializable; + +import org.archive.util.fingerprint.LongFPSet; + +import st.ata.util.FPGenerator; + +/** + * UriUniqFilter storing 64-bit UURI fingerprints, using an internal LongFPSet + * instance. + * + * The passed LongFPSet internal instance may be disk or memory based. Accesses + * to the underlying LongFPSet are synchronized. + * + * @author gojomo + */ +public class FPUriUniqFilter extends SetBasedUriUniqFilter +implements Serializable { + private static final long serialVersionUID = 1L;; + + private transient FPGenerator fpgen = FPGenerator.std64; + + LongFPSet fpset; + public LongFPSet getFpset() { + return this.fpset; + } + public void setFpset(LongFPSet fpset) { + this.fpset = fpset; + } + + /** + * Create FPUriUniqFilter wrapping given long set + * + * @param fpset + */ + public FPUriUniqFilter(LongFPSet fpset) { + this.fpset = fpset; + } + + + public FPUriUniqFilter() { + } + + private long getFp(CharSequence canonical) { + return fpgen.fp(canonical); + } + + protected boolean setAdd(CharSequence uri) { + return fpset.add(getFp(uri)); + } + + protected long setCount() { + return fpset.count(); + } + + protected boolean setRemove(CharSequence uri) { + return fpset.remove(getFp(uri)); + } +} \ No newline at end of file diff --git a/engine/src/main/java/org/archive/crawler/util/LogReader.java b/engine/src/main/java/org/archive/crawler/util/LogReader.java new file mode 100644 index 00000000..218acee0 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/util/LogReader.java @@ -0,0 +1,881 @@ +/* Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.crawler.util; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.RandomAccessFile; +import java.text.DecimalFormat; +import java.text.NumberFormat; +import java.util.LinkedList; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +import org.archive.io.CompositeFileReader; +import org.archive.util.ArchiveUtils; + +/** + * This class contains a variety of methods for reading log files (or other text + * files containing repeated lines with similar information). + *

      + * All methods are static. + * + * @author Kristinn Sigurdsson + */ + +public class LogReader +{ + /** + * Returns the entire file. Useful for smaller files. + * + * @param aFileName a file name + * @return The String representation of the entire file. + * Null is returned if errors occur (file not found or io exception) + */ + public static String get(String aFileName){ + try { + return get(new FileReader(aFileName)); + } catch (FileNotFoundException e) { + e.printStackTrace(); + return null; + } + } + + /** + * Reads entire contents of reader, returns as string. + * + * @param reader + * @return String of entire contents; null for any error. + */ + public static String get(InputStreamReader reader){ + StringBuffer ret = new StringBuffer(); + try{ + BufferedReader bf = new BufferedReader(reader, 8192); + + String line = null; + while ((line = bf.readLine()) != null) { + ret.append(line); + ret.append("\n"); + } + } catch(IOException e){ + e.printStackTrace(); + return null; + } + return ret.toString(); + } + + /** + * Gets a portion of a log file. Starting at a given line number and the n-1 + * lines following that one or until the end of the log if that is reached + * first. + * + * @param aFileName The filename of the log/file + * @param lineNumber The number of the first line to get (if larger then the + * file an empty string will be returned) + * @param n How many lines to return (total, including the one indicated by + * lineNumber). If smaller then 1 then an empty string + * will be returned. + * + * @return An array of two strings is returned. At index 0 a portion of the + * file starting at lineNumber and reaching lineNumber+n is located. + * At index 1 there is an informational string about how large a + * segment of the file is being returned. + * Null is returned if errors occur (file not found or io exception) + */ + public static String[] get(String aFileName, int lineNumber, int n) + { + File f = new File(aFileName); + long logsize = f.length(); + try { + return get(new FileReader(aFileName),lineNumber,n,logsize); + } catch (FileNotFoundException e) { + e.printStackTrace(); + return null; + } + } + + /** + * Gets a portion of a log spread across a numbered series of files. + * + * Starting at a given line number and the n-1 lines following that + * one or until the end of the log if that is reached + * first. + * + * @param aFileName The filename of the log/file + * @param lineNumber The number of the first line to get (if larger then the + * file an empty string will be returned) + * @param n How many lines to return (total, including the one indicated by + * lineNumber). If smaller then 1 then an empty string + * will be returned. + * + * @return An array of two strings is returned. At index 0 a portion of the + * file starting at lineNumber and reaching lineNumber+n is located. + * At index 1 there is an informational string about how large a + * segment of the file is being returned. + * Null is returned if errors occur (file not found or io exception) + */ + public static String[] getFromSeries(String aFileName, int lineNumber, int n) + { + File f = new File(aFileName); + long logsize = f.length(); + try { + return get(seriesReader(aFileName),lineNumber,n,logsize); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + + public static String buildDisplayingHeader(int len, long logsize) + { + double percent = 0.0; + if (logsize != 0) { + percent = ((double) len/logsize) * 100; + } + return "Displaying: " + ArchiveUtils.doubleToString(percent,1) + + "% of " + ArchiveUtils.formatBytesForDisplay(logsize); + } + + /** + * Gets a portion of a log file. Starting at a given line number and the n-1 + * lines following that one or until the end of the log if that is reached + * first. + * + * @param reader source to scan for lines + * @param lineNumber The number of the first line to get (if larger then the + * file an empty string will be returned) + * @param n How many lines to return (total, including the one indicated by + * lineNumber). If smaller then 1 then an empty string + * will be returned. + * + * @param logsize total size of source + * @return An array of two strings is returned. At index 0 a portion of the + * file starting at lineNumber and reaching lineNumber+n is located. + * At index 1 there is an informational string about how large a + * segment of the file is being returned. + * Null is returned if errors occur (file not found or io exception) + */ + public static String[] get(InputStreamReader reader, + int lineNumber, + int n, + long logsize) + { + StringBuffer ret = new StringBuffer(); + String info = null; + try{ + BufferedReader bf = new BufferedReader(reader, 8192); + + String line = null; + int i=1; + while ((line = bf.readLine()) != null) { + if(i >= lineNumber && i < (lineNumber+n)) + { + ret.append(line); + ret.append('\n'); + } else if( i >= (lineNumber+n)){ + break; + } + i++; + } + info = buildDisplayingHeader(ret.length(), logsize); + }catch(IOException e){ + e.printStackTrace(); + return null; + } + String[] tmp = {ret.toString(),info}; + return tmp; + } + + /** + * Return the line number of the first line in the + * log/file that matches a given regular expression. + * + * @param aFileName The filename of the log/file + * @param regExpr The regular expression that is to be used + * @return The line number (counting from 1, not zero) of the first line + * that matches the given regular expression. -1 is returned if no + * line matches the regular expression. -1 also is returned if + * errors occur (file not found, io exception etc.) + */ + public static int findFirstLineContaining(String aFileName, String regExpr) + { + try { + return findFirstLineContaining(new FileReader(aFileName), regExpr); + } catch (FileNotFoundException e) { + e.printStackTrace(); + return -1; + } + } + + /** + * Return the line number of the first line in the + * log/file that begins with the given string. + * + * @param aFileName The filename of the log/file + * @param prefix The prefix string to match + * @return The line number (counting from 1, not zero) of the first line + * that matches the given regular expression. -1 is returned if no + * line matches the regular expression. -1 also is returned if + * errors occur (file not found, io exception etc.) + */ + public static int findFirstLineBeginningFromSeries(String aFileName, + String prefix) + { + try { + return findFirstLineBeginning(seriesReader(aFileName), prefix); + } catch (IOException e) { + e.printStackTrace(); + return -1; + } + } + + /** + * Return the line number of the first line in the + * log/file that that begins with the given string. + * + * @param reader The reader of the log/file + * @param prefix The prefix string to match + * @return The line number (counting from 1, not zero) of the first line + * that matches the given regular expression. -1 is returned if no + * line matches the regular expression. -1 also is returned if + * errors occur (file not found, io exception etc.) + */ + public static int findFirstLineBeginning(InputStreamReader reader, + String prefix) + { + + try{ + BufferedReader bf = new BufferedReader(reader, 8192); + + String line = null; + int i = 1; + while ((line = bf.readLine()) != null) { + if(line.startsWith(prefix)){ + // Found a match + return i; + } + i++; + } + } catch(IOException e){ + e.printStackTrace(); + } + return -1; + } + + /** + * Return the line number of the first line in the + * log/file that matches a given regular expression. + * + * @param aFileName The filename of the log/file + * @param regExpr The regular expression that is to be used + * @return The line number (counting from 1, not zero) of the first line + * that matches the given regular expression. -1 is returned if no + * line matches the regular expression. -1 also is returned if + * errors occur (file not found, io exception etc.) + */ + public static int findFirstLineContainingFromSeries(String aFileName, + String regExpr) + { + try { + return findFirstLineContaining(seriesReader(aFileName), regExpr); + } catch (IOException e) { + e.printStackTrace(); + return -1; + } + } + + /** + * Return the line number of the first line in the + * log/file that matches a given regular expression. + * + * @param reader The reader of the log/file + * @param regExpr The regular expression that is to be used + * @return The line number (counting from 1, not zero) of the first line + * that matches the given regular expression. -1 is returned if no + * line matches the regular expression. -1 also is returned if + * errors occur (file not found, io exception etc.) + */ + public static int findFirstLineContaining(InputStreamReader reader, + String regExpr) + { + Pattern p = Pattern.compile(regExpr); + + try{ + BufferedReader bf = new BufferedReader(reader, 8192); + + String line = null; + int i = 1; + while ((line = bf.readLine()) != null) { + if(p.matcher(line).matches()){ + // Found a match + return i; + } + i++; + } + } catch(IOException e){ + e.printStackTrace(); + } + return -1; + } + + /** + * Returns all lines in a log/file matching a given regular expression. + * Possible to get lines immediately following the matched line. Also + * possible to have each line prepended by it's line number. + * + * @param aFileName The filename of the log/file + * @param regExpr The regular expression that is to be used + * @param addLines How many lines (in addition to the matched line) to add. + * A value less then 1 will mean that only the matched line + * will be included. If another matched line is hit before + * we reach this limit it will be included and this counter + * effectively reset for it. + * @param prependLineNumbers If true, then each line will be prepended by + * it's line number in the file. + * @param skipFirstMatches The first number of matches up to this value will + * be skipped over. + * @param numberOfMatches Once past matches that are to be skipped this many + * matches will be added to the return value. A + * value of 0 will cause all matching lines to be + * included. + * @return An array of two strings is returned. At index 0 tall lines in a + * log/file matching a given regular expression is located. + * At index 1 there is an informational string about how large a + * segment of the file is being returned. + * Null is returned if errors occur (file not found or io exception) + * If a PatternSyntaxException occurs, it's error message will be + * returned and the informational string will be empty (not null). + */ + public static String[] getByRegExpr(String aFileName, + String regExpr, + int addLines, + boolean prependLineNumbers, + int skipFirstMatches, + int numberOfMatches) { + try { + File f = new File(aFileName); + return getByRegExpr( + new FileReader(f), + regExpr, + addLines, + prependLineNumbers, + skipFirstMatches, + numberOfMatches, + f.length()); + } catch (FileNotFoundException e) { + e.printStackTrace(); + return null; + } + } + + /** + * Returns all lines in a log/file matching a given regular expression. + * Possible to get lines immediately following the matched line. Also + * possible to have each line prepended by it's line number. + * + * @param aFileName The filename of the log/file + * @param regExpr The regular expression that is to be used + * @param addLines How many lines (in addition to the matched line) to add. + * A value less then 1 will mean that only the matched line + * will be included. If another matched line is hit before + * we reach this limit it will be included and this counter + * effectively reset for it. + * @param prependLineNumbers If true, then each line will be prepended by + * it's line number in the file. + * @param skipFirstMatches The first number of matches up to this value will + * be skipped over. + * @param numberOfMatches Once past matches that are to be skipped this many + * matches will be added to the return value. A + * value of 0 will cause all matching lines to be + * included. + * @return An array of two strings is returned. At index 0 tall lines in a + * log/file matching a given regular expression is located. + * At index 1 there is an informational string about how large a + * segment of the file is being returned. + * Null is returned if errors occur (file not found or io exception) + * If a PatternSyntaxException occurs, it's error message will be + * returned and the informational string will be empty (not null). + */ + public static String[] getByRegExprFromSeries(String aFileName, + String regExpr, + int addLines, + boolean prependLineNumbers, + int skipFirstMatches, + int numberOfMatches) { + try { + File f = new File(aFileName); + return getByRegExpr( + seriesReader(aFileName), + regExpr, + addLines, + prependLineNumbers, + skipFirstMatches, + numberOfMatches, + f.length()); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + + /** + * Returns all lines in a log/file matching a given regular expression. + * Possible to get lines immediately following the matched line. Also + * possible to have each line prepended by it's line number. + * + * @param reader The reader of the log/file + * @param regExpr The regular expression that is to be used + * @param addLines How many lines (in addition to the matched line) to add. + * A value less then 1 will mean that only the matched line + * will be included. If another matched line is hit before + * we reach this limit it will be included and this counter + * effectively reset for it. + * @param prependLineNumbers If true, then each line will be prepended by + * it's line number in the file. + * @param skipFirstMatches The first number of matches up to this value will + * be skipped over. + * @param numberOfMatches Once past matches that are to be skipped this many + * matches will be added to the return value. A + * value of 0 will cause all matching lines to be + * included. + * @param logsize Size of the log in bytes + * @return An array of two strings is returned. At index 0 all lines in a + * log/file matching a given regular expression is located. + * At index 1 there is an informational string about how large a + * segment of the file is being returned. + * Null is returned if errors occur (file not found or io exception) + * If a PatternSyntaxException occurs, it's error message will be + * returned and the informational string will be empty (not null). + */ + public static String[] getByRegExpr(InputStreamReader reader, + String regExpr, + int addLines, + boolean prependLineNumbers, + int skipFirstMatches, + int numberOfMatches, + long logsize) { + StringBuffer ret = new StringBuffer(); + String info = ""; + + try{ + Pattern p = Pattern.compile(regExpr); + BufferedReader bf = new BufferedReader(reader, 8192); + + String line = null; + int i = 1; + boolean doAdd = false; + int addCount = 0; + long linesMatched = 0; + while ((line = bf.readLine()) != null) { + if(p.matcher(line).matches()){ + // Found a match + if(numberOfMatches > 0 && + linesMatched >= skipFirstMatches + numberOfMatches){ + // Ok, we are done. + break; + } + linesMatched++; + if(linesMatched > skipFirstMatches){ + if(prependLineNumbers){ + ret.append(i); + ret.append(". "); + } + ret.append(line); + ret.append("\n"); + doAdd = true; + addCount = 0; + } + } else if(doAdd) { + if(addCount < addLines){ + //Ok, still within addLines + linesMatched++; + if(prependLineNumbers){ + ret.append(i); + ret.append(". "); + } + ret.append(line); + ret.append("\n"); + }else{ + doAdd = false; + addCount = 0; + } + } + i++; + } + info = buildDisplayingHeader(ret.length(), logsize); + }catch(FileNotFoundException e){ + return null; + }catch(IOException e){ + e.printStackTrace(); + return null; + }catch(PatternSyntaxException e){ + ret = new StringBuffer(e.getMessage()); + } + String[] tmp = {ret.toString(),info}; + return tmp; + } + + /** + * Returns all lines in a log/file matching a given regular expression. + * Possible to get lines immediately following the matched line. Also + * possible to have each line prepended by it's line number. + * + * @param aFileName The filename of the log/file + * @param regExpr The regular expression that is to be used + * @param addLines Any lines following a match that begin with this + * string will also be included. We will stop including new + * lines once we hit the first that does not match. + * @param prependLineNumbers If true, then each line will be prepended by + * it's line number in the file. + * @param skipFirstMatches The first number of matches up to this value will + * be skipped over. + * @param numberOfMatches Once past matches that are to be skipped this many + * matches will be added to the return value. A + * value of 0 will cause all matching lines to be + * included. + * @return An array of two strings is returned. At index 0 tall lines in a + * log/file matching a given regular expression is located. + * At index 1 there is an informational string about how large a + * segment of the file is being returned. + * Null is returned if errors occur (file not found or io exception) + * If a PatternSyntaxException occurs, it's error message will be + * returned and the informational string will be empty (not null). + */ + public static String[] getByRegExpr(String aFileName, + String regExpr, + String addLines, + boolean prependLineNumbers, + int skipFirstMatches, + int numberOfMatches){ + try { + File f = new File(aFileName); + return getByRegExpr( + new FileReader(f), + regExpr, + addLines, + prependLineNumbers, + skipFirstMatches, + numberOfMatches, + f.length()); + } catch (FileNotFoundException e) { + e.printStackTrace(); + return null; + } + } + + /** + * Returns all lines in a log/file matching a given regular expression. + * Possible to get lines immediately following the matched line. Also + * possible to have each line prepended by it's line number. + * + * @param aFileName The filename of the log/file + * @param regExpr The regular expression that is to be used + * @param addLines Any lines following a match that begin with this + * string will also be included. We will stop including new + * lines once we hit the first that does not match. + * @param prependLineNumbers If true, then each line will be prepended by + * it's line number in the file. + * @param skipFirstMatches The first number of matches up to this value will + * be skipped over. + * @param numberOfMatches Once past matches that are to be skipped this many + * matches will be added to the return value. A + * value of 0 will cause all matching lines to be + * included. + * @return An array of two strings is returned. At index 0 tall lines in a + * log/file matching a given regular expression is located. + * At index 1 there is an informational string about how large a + * segment of the file is being returned. + * Null is returned if errors occur (file not found or io exception) + * If a PatternSyntaxException occurs, it's error message will be + * returned and the informational string will be empty (not null). + */ + public static String[] getByRegExprFromSeries(String aFileName, + String regExpr, + String addLines, + boolean prependLineNumbers, + int skipFirstMatches, + int numberOfMatches){ + try { + File f = new File(aFileName); + return getByRegExpr( + seriesReader(aFileName), + regExpr, + addLines, + prependLineNumbers, + skipFirstMatches, + numberOfMatches, + f.length()); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + + /** + * Returns all lines in a log/file matching a given regular expression. + * Possible to get lines immediately following the matched line. Also + * possible to have each line prepended by it's line number. + * + * @param reader The reader of the log/file + * @param regExpr The regular expression that is to be used + * @param addLines Any lines following a match that begin with this + * string will also be included. We will stop including new + * lines once we hit the first that does not match. + * @param prependLineNumbers If true, then each line will be prepended by + * it's line number in the file. + * @param skipFirstMatches The first number of matches up to this value will + * be skipped over. + * @param numberOfMatches Once past matches that are to be skipped this many + * matches will be added to the return value. A + * value of 0 will cause all matching lines to be + * included. + * @param logsize Size of the log in bytes + * @return An array of two strings is returned. At index 0 tall lines in a + * log/file matching a given regular expression is located. + * At index 1 there is an informational string about how large a + * segment of the file is being returned. + * Null is returned if errors occur (file not found or io exception) + * If a PatternSyntaxException occurs, it's error message will be + * returned and the informational string will be empty (not null). + */ + public static String[] getByRegExpr(InputStreamReader reader, + String regExpr, + String addLines, + boolean prependLineNumbers, + int skipFirstMatches, + int numberOfMatches, + long logsize) { + StringBuffer ret = new StringBuffer(); + String info = ""; + try{ + Matcher m = Pattern.compile(regExpr).matcher(""); + BufferedReader bf = new BufferedReader(reader, 8192); + + String line = null; + int i = 1; + boolean doAdd = false; + long linesMatched = 0; + while ((line = bf.readLine()) != null) { + m.reset(line); + if(m.matches()){ + // Found a match + if(numberOfMatches > 0 && + linesMatched >= skipFirstMatches + numberOfMatches){ + // Ok, we are done. + break; + } + linesMatched++; + if(linesMatched > skipFirstMatches){ + if(prependLineNumbers){ + ret.append(i); + ret.append(". "); + } + ret.append(line); + ret.append("\n"); + doAdd = true; + } + } else if(doAdd) { + if(line.indexOf(addLines)==0){ + linesMatched++; + //Ok, line begins with 'addLines' + if(prependLineNumbers){ + ret.append(i); + ret.append(". "); + } + ret.append(line); + ret.append("\n"); + }else{ + doAdd = false; + } + } + i++; + } + info = buildDisplayingHeader(ret.length(), logsize); + }catch(FileNotFoundException e){ + return null; + }catch(IOException e){ + e.printStackTrace(); + return null; + }catch(PatternSyntaxException e){ + ret = new StringBuffer(e.getMessage()); + } + String[] tmp = {ret.toString(),info}; + return tmp; + } + + /** + * Implementation of a unix-like 'tail' command + * + * @param aFileName a file name String + * @return An array of two strings is returned. At index 0 the String + * representation of at most 10 last lines is located. + * At index 1 there is an informational string about how large a + * segment of the file is being returned. + * Null is returned if errors occur (file not found or io exception) + */ + public static String[] tail(String aFileName) { + return tail(aFileName, 10); + } + + /** + * Implementation of a unix-like 'tail -n' command + * + * @param aFileName a file name String + * @param n int number of lines to be returned + * @return An array of two strings is returned. At index 0 the String + * representation of at most n last lines is located. + * At index 1 there is an informational string about how large a + * segment of the file is being returned. + * Null is returned if errors occur (file not found or io exception) + */ + public static String[] tail(String aFileName, int n) { + try { + return tail(new RandomAccessFile(new File(aFileName),"r"),n); + } catch (FileNotFoundException e) { + e.printStackTrace(); + return null; + } + } + + /** + * Implementation of a unix-like 'tail -n' command + * + * @param raf a RandomAccessFile to tail + * @param n int number of lines to be returned + * @return An array of two strings is returned. At index 0 the String + * representation of at most n last lines is located. + * At index 1 there is an informational string about how large a + * segment of the file is being returned. + * Null is returned if errors occur (file not found or io exception) + */ + public static String[] tail(RandomAccessFile raf, int n) { + int BUFFERSIZE = 1024; + long pos; + long endPos; + long lastPos; + int numOfLines = 0; + String info=null; + byte[] buffer = new byte[BUFFERSIZE]; + StringBuffer sb = new StringBuffer(); + try { + endPos = raf.length(); + lastPos = endPos; + + // Check for non-empty file + // Check for newline at EOF + if (endPos > 0) { + byte[] oneByte = new byte[1]; + raf.seek(endPos - 1); + raf.read(oneByte); + if ((char) oneByte[0] != '\n') { + numOfLines++; + } + } + + do { + // seek back BUFFERSIZE bytes + // if length of the file if less then BUFFERSIZE start from BOF + pos = 0; + if ((lastPos - BUFFERSIZE) > 0) { + pos = lastPos - BUFFERSIZE; + } + raf.seek(pos); + // If less then BUFFERSIZE avaliable read the remaining bytes + if ((lastPos - pos) < BUFFERSIZE) { + int remainer = (int) (lastPos - pos); + buffer = new byte[remainer]; + } + raf.readFully(buffer); + // in the buffer seek back for newlines + for (int i = buffer.length - 1; i >= 0; i--) { + if ((char) buffer[i] == '\n') { + numOfLines++; + // break if we have last n lines + if (numOfLines > n) { + pos += (i + 1); + break; + } + } + } + // reset last postion + lastPos = pos; + } while ((numOfLines <= n) && (pos != 0)); + + // print last n line starting from last postion + for (pos = lastPos; pos < endPos; pos += buffer.length) { + raf.seek(pos); + if ((endPos - pos) < BUFFERSIZE) { + int remainer = (int) (endPos - pos); + buffer = new byte[remainer]; + } + raf.readFully(buffer); + sb.append(new String(buffer)); + } + + info = buildDisplayingHeader(sb.length(), raf.length()); + } catch (FileNotFoundException e) { + sb = null; + } catch (IOException e) { + e.printStackTrace(); + sb = null; + } finally { + try { + if (raf != null) { + raf.close(); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + if(sb==null){ + return null; + } + String[] tmp = {sb.toString(),info}; + return tmp; + } + + /** + * @param fileName + * @return + * @throws IOException + */ + private static CompositeFileReader seriesReader(String fileName) + throws IOException { + LinkedList filenames = new LinkedList(); + int seriesNumber = 1; + NumberFormat fmt = new DecimalFormat("00000"); + String predecessorFilename = + fileName + + fmt.format(seriesNumber); + while((new File(predecessorFilename)).exists()) { + filenames.add(new File(predecessorFilename)); + seriesNumber++; + predecessorFilename = + fileName + + fmt.format(seriesNumber); + } + filenames.add(new File(fileName)); // add current file + return new CompositeFileReader(filenames); + } +} diff --git a/engine/src/main/java/org/archive/crawler/util/LogUtils.java b/engine/src/main/java/org/archive/crawler/util/LogUtils.java new file mode 100644 index 00000000..d425a732 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/util/LogUtils.java @@ -0,0 +1,84 @@ +/* LogUtils + * + * $Id$ + * + * Created on Jun 8, 2005 + * + * Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ +package org.archive.crawler.util; + +import java.io.File; +import java.lang.reflect.Constructor; +import java.util.logging.FileHandler; +import java.util.logging.Formatter; +import java.util.logging.Logger; + +import org.archive.util.PropertyUtils; + +/** + * Logging utils. + * @author stack + */ +public class LogUtils { + /** + * Creates a file logger that use heritrix.properties file logger + * configuration. + * Change the java.util.logging.FileHandler.* properties in + * heritrix.properties to change file handler properties. + * Use this method if you want a class to log to its own file + * rather than use default (console) logger. + * @param logsDir Directory in which to write logs. + * @param baseName Base name to use for log file (Will have + * java.util.logging.FileHandler.pattern or '.log' for suffix). + * @param logger Logger whose handler we'll replace with the + * file handler created herein. + */ + public static FileHandler createFileLogger(File logsDir, String baseName, + Logger logger) { + int limit = + PropertyUtils.getIntProperty("java.util.logging.FileHandler.limit", + 1024 * 1024 * 1024 * 1024); + int count = + PropertyUtils.getIntProperty("java.util.logging.FileHandler.count", 1); + try { + String tmp = + System.getProperty("java.util.logging.FileHandler.pattern"); + File logFile = new File(logsDir, baseName + + ((tmp != null && tmp.length() > 0)? tmp: ".log")); + FileHandler fh = new FileHandler(logFile.getAbsolutePath(), limit, + count, true); + // Manage the formatter to use. + tmp = System.getProperty("java.util.logging.FileHandler.formatter"); + if (tmp != null && tmp.length() > 0) { + Constructor co = Class.forName(tmp). + getConstructor(new Class[] {}); + Formatter f = (Formatter) co.newInstance(new Object[] {}); + fh.setFormatter(f); + } + logger.addHandler(fh); + logger.setUseParentHandlers(false); + return fh; + } catch (Exception e) { + logger.severe("Failed customization of logger: " + e.getMessage()); + return null; + } + } +} diff --git a/engine/src/main/java/org/archive/crawler/util/Logs.java b/engine/src/main/java/org/archive/crawler/util/Logs.java new file mode 100644 index 00000000..fa233392 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/util/Logs.java @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2007 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Logs.java + * + * Created on June 14, 2007 + * + * $Id:$ + */ +package org.archive.crawler.util; + + +/** + * Enumerates existing Heritrix logs + * + * @author Kristinn Sigurdsson + */ +public enum Logs{ + // TODO: This enum belongs in the heritrix sub project + CRAWL ("crawl.log"), + ALERTS ("alerts.log"), + PROGRESS_STATISTICS ("progress-statistics.log"), + RUNTIME_ERRORS ("runtime-errors.log"), + NONFATAL_ERRORS ("nonfatal-errors.log"), + URI_ERRORS ("uri-errors.log"); + + String filename; + + Logs(String filename){ + this.filename = filename; + } + + public String getFilename(){ + return filename; + } +} \ No newline at end of file diff --git a/engine/src/main/java/org/archive/crawler/util/MemFPMergeUriUniqFilter.java b/engine/src/main/java/org/archive/crawler/util/MemFPMergeUriUniqFilter.java new file mode 100644 index 00000000..50621a05 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/util/MemFPMergeUriUniqFilter.java @@ -0,0 +1,69 @@ +/* MemFPMergeUriUniqFilter +* +* $Id$ +* +* Created on Dec 14, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.crawler.util; + +import it.unimi.dsi.fastutil.longs.LongArrayList; +import it.unimi.dsi.fastutil.longs.LongIterator; + +/** + * Crude all-in-memory FP-merging UriUniqFilter. + * + * @author gojomo + */ +public class MemFPMergeUriUniqFilter extends FPMergeUriUniqFilter { + protected LongArrayList allFps = new LongArrayList(); + protected LongArrayList newFps; + + /* (non-Javadoc) + * @see org.archive.crawler.util.FPMergeUriUniqFilter#beginFpMerge() + */ + protected LongIterator beginFpMerge() { + newFps = new LongArrayList((int) (allFps.size()+(pending()/2))); + return allFps.iterator(); + } + + /* (non-Javadoc) + * @see org.archive.crawler.util.FPMergeUriUniqFilter#addNewFp(java.lang.Long) + */ + protected void addNewFp(long currFp) { + newFps.add(currFp); + } + + /* (non-Javadoc) + * @see org.archive.crawler.util.FPMergeUriUniqFilter#finishFpMerge() + */ + protected void finishFpMerge() { + allFps = newFps; + newFps = null; + } + + /* (non-Javadoc) + * @see org.archive.crawler.datamodel.UriUniqFilter#count() + */ + public long count() { + return allFps.size(); + } + +} diff --git a/engine/src/main/java/org/archive/crawler/util/MemUriUniqFilter.java b/engine/src/main/java/org/archive/crawler/util/MemUriUniqFilter.java new file mode 100644 index 00000000..306fb335 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/util/MemUriUniqFilter.java @@ -0,0 +1,57 @@ +/* Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * MemUURISet.java + * Created on Sep 15, 2003 + * + * $Header$ + */ +package org.archive.crawler.util; + +import java.util.HashSet; + +/** + * A purely in-memory UriUniqFilter based on a HashSet, which remembers + * every full URI string it sees. + * + * @author gojomo + * + */ +public class MemUriUniqFilter +extends SetBasedUriUniqFilter { + private static final long serialVersionUID = 1L; + HashSet hashSet; + + protected synchronized boolean setAdd(CharSequence uri) { + return hashSet.add(uri); + } + protected synchronized boolean setRemove(CharSequence uri) { + return hashSet.remove(uri); + } + protected synchronized long setCount() { + return (long)hashSet.size(); + } + + /* (non-Javadoc) + * @see org.archive.crawler.util.UriUniqFilterImpl#createUriSet() + */ + protected void createUriSet() { + hashSet = new HashSet(); + } + +} diff --git a/engine/src/main/java/org/archive/crawler/util/NoopUriUniqFilter.java b/engine/src/main/java/org/archive/crawler/util/NoopUriUniqFilter.java new file mode 100644 index 00000000..e92d2d6d --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/util/NoopUriUniqFilter.java @@ -0,0 +1,55 @@ +/* Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * MemUURISet.java + * Created on Sep 15, 2003 + * + * $Header$ + */ +package org.archive.crawler.util; + + +/** + * A UriUniqFilter that doesn't actually provide any uniqueness + * filter on presented items: all are passed through. + * + * @author gojomo + * + */ +public class NoopUriUniqFilter +extends SetBasedUriUniqFilter { + private static final long serialVersionUID = 1L; + + protected synchronized boolean setAdd(CharSequence uri) { + return true; // always consider as new + } + protected synchronized boolean setRemove(CharSequence uri) { + return true; // always consider as succeeded + } + protected synchronized long setCount() { + return -1; // return nonsense value + } + + /* (non-Javadoc) + * @see org.archive.crawler.util.UriUniqFilterImpl#createUriSet() + */ + protected void createUriSet() { + // do nothing + } + +} diff --git a/engine/src/main/java/org/archive/crawler/util/RecoveryLogMapper.java b/engine/src/main/java/org/archive/crawler/util/RecoveryLogMapper.java new file mode 100644 index 00000000..357f182b --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/util/RecoveryLogMapper.java @@ -0,0 +1,324 @@ +/* RecoveryLogMapper.java +* +* $Id$ +* +* Created on Mar 7, 2005 +* +* Copyright (C) 2005 Mike Schwartz. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +/** + * Parses a Heritrix recovery log file (recover.gz), and builds maps + * that allow a caller to look up any seed URL and get back an Iterator of all + * URLs successfully crawled from given seed. + * + * Also allows lookup on any crawled + * URL to find the seed URL from which the crawler reached that URL (through 1 + * or more discovered URL hops, which are collapsed in this lookup). + * + *

      This code creates some fairly large collections (proprotionate in size to + * # discovered URLs) so make sure you allocate + * it a large heap to work in. It also takes a while to process a recover log. + *

      See {@link #main()} method at end for test/demo code. + * @author Mike Schwartz, schwartz at CodeOnTheRoad dot com + */ +package org.archive.crawler.util; + +import org.archive.crawler.frontier.FrontierJournal; + +import java.io.File; +import java.io.LineNumberReader; +import java.io.PrintWriter; +import java.io.FileOutputStream; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; + +public class RecoveryLogMapper { + private static final char LOG_LINE_START_CHAR = + FrontierJournal.F_ADD.charAt(0); + private static final Logger logger = + Logger.getLogger(RecoveryLogMapper.class.getName()); + private PrintWriter seedNotFoundPrintWriter = null; + + /** + * Tracks seed for each crawled URL + */ + private Map crawledUrlToSeedMap + = new HashMap(); + + /** + * Maps seed URLs to Set of discovered URLs + */ + private Map> seedUrlToDiscoveredUrlsMap + = new HashMap>(); + + /** + * Tracks which URLs were successfully crawled + */ + private Set successfullyCrawledUrls = new HashSet(); + + /** + * Normal constructor - if encounter not-found seeds while loading + * recoverLogFileName, will throw throw SeedUrlNotFoundException. + * Use {@link #RecoveryLogMapper(String)} if you want to just log + * such cases and keep going. (Those should not happen if the + * recover log is written correctly, but we see them in pratice.) + * @param recoverLogFileName + * @throws java.io.FileNotFoundException + * @throws java.io.IOException + * @throws SeedUrlNotFoundException + */ + public RecoveryLogMapper(String recoverLogFileName) + throws java.io.FileNotFoundException, java.io.IOException, + SeedUrlNotFoundException { + load(recoverLogFileName); + } + + /** + * Constructor to use if you want to allow not-found seeds, logging + * them to seedNotFoundLogFileName. In contrast, {@link + * #RecoveryLogMapper(String)} will throw SeedUrlNotFoundException + * when a seed isn't found. + * @param recoverLogFileName + * @param seedNotFoundLogFileName + */ + public RecoveryLogMapper(String recoverLogFileName, + String seedNotFoundLogFileName) + throws java.io.FileNotFoundException, java.io.IOException, + SeedUrlNotFoundException { + seedNotFoundPrintWriter = new PrintWriter(new FileOutputStream( + seedNotFoundLogFileName)); + load(recoverLogFileName); + } + + protected void load(String recoverLogFileName) + throws java.io.FileNotFoundException, java.io.IOException, + SeedUrlNotFoundException { + LineNumberReader reader = new LineNumberReader(FrontierJournal. + getBufferedReader(new File(recoverLogFileName))); + String curLine = null; + while ((curLine = reader.readLine()) != null) { + if (curLine.length() == 0 + || curLine.charAt(0) != LOG_LINE_START_CHAR) { + continue; + } + String args[] = curLine.split("\\s+"); + int curLineNumWords = args.length; + String firstUrl = args[1]; + // Ignore DNS log entries + if (firstUrl.startsWith("dns:")) { + continue; + } + if (curLine.startsWith(FrontierJournal.F_ADD)) { + // Seed URL + if (curLineNumWords == 2) { + if (logger.isLoggable(Level.FINE)) { + logger.fine("F_ADD with 2 words --> seed URL (" + + firstUrl + ")"); + } + // Add seed the first time we find it + if (seedUrlToDiscoveredUrlsMap.get(firstUrl) == null) { + seedUrlToDiscoveredUrlsMap.put(firstUrl, + new HashSet()); + } + } else { + // URL found via an earlier seeded / discovered URL + // Look for the seed from which firstUrlString came, so + // we can collapse new URLString back to it + String viaUrl = args[curLineNumWords - 1]; + if (logger.isLoggable(Level.FINE)) { + logger.fine("F_ADD with 3+ words --> new URL " + + firstUrl + " via URL " + viaUrl); + } + String seedForFirstUrl = + (String) crawledUrlToSeedMap.get(viaUrl); + // viaUrlString is a seed URL + if (seedForFirstUrl == null) { + if (logger.isLoggable(Level.FINE)) { + logger.fine("\tvia URL is a seed"); + } + crawledUrlToSeedMap.put(firstUrl, viaUrl); + seedForFirstUrl = viaUrl; + } else { + if (logger.isLoggable(Level.FINE)) { + logger.fine("\tvia URL discovered via seed URL " + + seedForFirstUrl); + } + // Collapse + crawledUrlToSeedMap.put(firstUrl, seedForFirstUrl); + } + Set theSeedUrlList = + seedUrlToDiscoveredUrlsMap.get(seedForFirstUrl); + if (theSeedUrlList == null) { + String message = "recover log " + + recoverLogFileName + " at line " + + reader.getLineNumber() + + " listed F+ URL (" + viaUrl + + ") for which found no seed list."; + if (seedNotFoundPrintWriter != null) { + seedNotFoundPrintWriter.println(message); + } else { + throw new SeedUrlNotFoundException(message); + } + } else { + theSeedUrlList.add(firstUrl); + } + } + } else if (curLine.startsWith(FrontierJournal.F_SUCCESS)) { + if (logger.isLoggable(Level.FINE)) { + logger.fine("F_SUCCESS for URL " + firstUrl); + } + successfullyCrawledUrls.add(firstUrl); + } + } + reader.close(); + if (seedNotFoundPrintWriter != null) { + seedNotFoundPrintWriter.close(); + } + } + + /** + * Returns seed for urlString (null if seed not found). + * @param urlString + * @return Seed. + */ + public String getSeedForUrl(String urlString) { + return (seedUrlToDiscoveredUrlsMap.get(urlString) != null)? + urlString: crawledUrlToSeedMap.get(urlString); + } + + /** + * @return Returns the seedUrlToDiscoveredUrlsMap. + */ + public Map> getSeedUrlToDiscoveredUrlsMap() { + return this.seedUrlToDiscoveredUrlsMap; + } + + /** + * @return Returns the successfullyCrawledUrls. + */ + public Set getSuccessfullyCrawledUrls() { + return this.successfullyCrawledUrls; + } + + /** + * @return Returns the logger. + */ + public static Logger getLogger() { + return logger; + } + + private class SuccessfullyCrawledURLsIterator + implements Iterator { + private String nextValue = null; + private Iterator discoveredUrlsIterator; + + public SuccessfullyCrawledURLsIterator(String seedUrlString) + throws SeedUrlNotFoundException { + Set discoveredUrlList = + (Set)getSeedUrlToDiscoveredUrlsMap().get(seedUrlString); + if (discoveredUrlList == null) { + throw new SeedUrlNotFoundException("Seed URL " + + seedUrlString + " not found in seed list"); + } + discoveredUrlsIterator = discoveredUrlList.iterator(); + } + + /** + * Idempotent method (because of null check on nextValue). + */ + private void populateNextValue() { + while (nextValue == null & discoveredUrlsIterator.hasNext()) { + String curDiscoveredUrl = discoveredUrlsIterator.next(); + boolean succCrawled = getSuccessfullyCrawledUrls(). + contains(curDiscoveredUrl); + if (getLogger().isLoggable(Level.FINE)) { + getLogger().fine("populateNextValue: curDiscoveredUrl=" + + curDiscoveredUrl + ", succCrawled=" + + succCrawled); + } + if (succCrawled) + nextValue = curDiscoveredUrl; + } + } + + public boolean hasNext() { + populateNextValue(); + return (nextValue != null); + } + + public String next() { + populateNextValue(); + String returnValue = nextValue; + nextValue = null; + return returnValue; + } + + /** + * Remove operation is unsupported in this Iterator + * (will throw UnsupportedOperationException if called). + */ + public void remove() { + throw new UnsupportedOperationException( + "SuccessfullyCrawledURLsIterator.remove: not supported."); + } + } + + public Iterator getIteratorOfURLsSuccessfullyCrawledFromSeedUrl( + String seedUrlString) throws SeedUrlNotFoundException { + return new SuccessfullyCrawledURLsIterator(seedUrlString); + } + + public Collection getSeedCollection() { + return seedUrlToDiscoveredUrlsMap.keySet(); + } + + public static void main(String args[]) { + if (args.length < 1) { + System.out.println("Usage: RecoveryLogMapper recoverLogFileName"); + Runtime.getRuntime().exit(-1); + } + String recoverLogFileName = args[0]; + try { + RecoveryLogMapper myRecoveryLogMapper = + new RecoveryLogMapper(recoverLogFileName); + for (String curSeedUrl: myRecoveryLogMapper.getSeedCollection()) { + System.out.println("URLs successfully crawled from seed URL " + + curSeedUrl); + Iterator iteratorOfUrlsCrawledFromSeedUrl = + myRecoveryLogMapper. + getIteratorOfURLsSuccessfullyCrawledFromSeedUrl( + curSeedUrl); + while (iteratorOfUrlsCrawledFromSeedUrl.hasNext()) { + String curCrawledUrlString = + (String)iteratorOfUrlsCrawledFromSeedUrl.next(); + System.out.println(" -> " + curCrawledUrlString); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/engine/src/main/java/org/archive/crawler/util/SeedUrlNotFoundException.java b/engine/src/main/java/org/archive/crawler/util/SeedUrlNotFoundException.java new file mode 100644 index 00000000..af51eb1f --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/util/SeedUrlNotFoundException.java @@ -0,0 +1,37 @@ +/* SeedUrlNotFoundException +* +* $Id$ +* +* Created on Mar 9, 2005 +* +* Copyright (C) 2005 Mike Schwartz. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.crawler.util; + +/** + * @author Mike Schwartz, schwartz at CodeOnTheRoad dot com + */ +public class SeedUrlNotFoundException extends Exception { + + private static final long serialVersionUID = 2515927240634523493L; + + public SeedUrlNotFoundException(String message) { + super(message); + } +} diff --git a/engine/src/main/java/org/archive/crawler/util/SetBasedUriUniqFilter.java b/engine/src/main/java/org/archive/crawler/util/SetBasedUriUniqFilter.java new file mode 100644 index 00000000..73771c23 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/util/SetBasedUriUniqFilter.java @@ -0,0 +1,141 @@ +/* UriUniqFilterImpl +* +* $Id$ +* +* Created on Sep 29, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.crawler.util; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.PrintWriter; +import java.io.Serializable; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.archive.crawler.datamodel.CrawlURI; +import org.archive.crawler.datamodel.UriUniqFilter; + +/** + * UriUniqFilter based on an underlying UriSet (essentially a Set). + * + * @author gojomo + */ +public abstract class SetBasedUriUniqFilter implements UriUniqFilter, Serializable { + private static Logger LOGGER = + Logger.getLogger(SetBasedUriUniqFilter.class.getName()); + + protected CrawlUriReceiver receiver; + protected PrintWriter profileLog; + protected long duplicateCount = 0; + protected long duplicatesAtLastSample = 0; + + public SetBasedUriUniqFilter() { + super(); + String profileLogFile = + System.getProperty(SetBasedUriUniqFilter.class.getName() + + ".profileLogFile"); + if (profileLogFile != null) { + setProfileLog(new File(profileLogFile)); + } + } + + protected abstract boolean setAdd(CharSequence key); + + protected abstract boolean setRemove(CharSequence key); + + protected abstract long setCount(); + + public long count() { + return setCount(); + } + + public long pending() { + // no items pile up in this implementation + return 0; + } + + public void setDestination(CrawlUriReceiver receiver) { + this.receiver = receiver; + } + + protected void profileLog(String key) { + if (profileLog != null) { + profileLog.println(key); + } + } + + public void add(String key, CrawlURI value) { + profileLog(key); + if (setAdd(key)) { + this.receiver.receive(value); + if (setCount() % 50000 == 0) { + LOGGER.log(Level.FINE, "count: " + setCount() + " totalDups: " + + duplicateCount + " recentDups: " + + (duplicateCount - duplicatesAtLastSample)); + duplicatesAtLastSample = duplicateCount; + } + } else { + duplicateCount++; + } + } + + public void addNow(String key, CrawlURI value) { + add(key, value); + } + + public void addForce(String key, CrawlURI value) { + profileLog(key); + setAdd(key); + this.receiver.receive(value); + } + + public void note(String key) { + profileLog(key); + setAdd(key); + } + + public void forget(String key, CrawlURI value) { + setRemove(key); + } + + public long requestFlush() { + // unnecessary; all actions with set-based uniqfilter are immediate + return 0; + } + + public void close() { + if (profileLog != null) { + profileLog.close(); + } + } + + public void setProfileLog(File logfile) { + try { + profileLog = new PrintWriter(new BufferedOutputStream( + new FileOutputStream(logfile))); + } catch (FileNotFoundException e) { + throw new RuntimeException(e); + } + } +} diff --git a/engine/src/main/java/org/archive/crawler/util/Sorts.java b/engine/src/main/java/org/archive/crawler/util/Sorts.java new file mode 100644 index 00000000..fada83a8 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/util/Sorts.java @@ -0,0 +1,49 @@ +/* Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Created on Jul 21, 2003 + * + * To change the template for this generated file go to + * Window>Preferences>Java>Code Generation>Code and Comments + */ +package org.archive.crawler.util; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; + +public class Sorts { + + // Sorts by value not key + public static StringIntPair[] sortStringIntHashMap (HashMap hm){ + String[] keys = hm.keySet().toArray(new String[hm.size()]); + Integer[] values = hm.values().toArray(new Integer[hm.size()]); + + ArrayList unsortedList = new ArrayList(); + + for (int i = 0; i < keys.length; i++) + unsortedList.add(i, new StringIntPair(keys[i], values[i])); + + StringIntPair[] sortedArray + = unsortedList.toArray(new StringIntPair[unsortedList.size()]); + Arrays.sort(sortedArray, new StringIntPairComparator()); + + return sortedArray; + } + +} diff --git a/engine/src/main/java/org/archive/crawler/util/StringIntPair.java b/engine/src/main/java/org/archive/crawler/util/StringIntPair.java new file mode 100644 index 00000000..ea63accc --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/util/StringIntPair.java @@ -0,0 +1,63 @@ +/* Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Created on Jul 21, 2003 + * + * To change the template for this generated file go to + * Window>Preferences>Java>Code Generation>Code and Comments + */ +package org.archive.crawler.util; + +/** + * @author Administrator + * + * To change the template for this generated type comment go to + * Window>Preferences>Java>Code Generation>Code and Comments + */ +public class StringIntPair { + private String stringValue; + private int intValue; + + public StringIntPair(String s, int i){ + stringValue = s; + intValue = i; + } + + public int getIntValue() { + return intValue; + } + + public String getStringValue() { + return stringValue; + } + + /** + * @param i + */ + public void setIntValue(int i) { + intValue = i; + } + + /** + * @param string + */ + public void setStringValue(String string) { + stringValue = string; + } + +} diff --git a/engine/src/main/java/org/archive/crawler/util/StringIntPairComparator.java b/engine/src/main/java/org/archive/crawler/util/StringIntPairComparator.java new file mode 100644 index 00000000..95c3ba40 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/util/StringIntPairComparator.java @@ -0,0 +1,48 @@ +/* Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Created on Jul 21, 2003 + * + * To change the template for this generated file go to + * Window>Preferences>Java>Code Generation>Code and Comments + */ +package org.archive.crawler.util; + +import java.util.Comparator; + +/** + * @author Igor Ranitovic + * + * + * */ +public class StringIntPairComparator implements Comparator { + + public StringIntPairComparator (){ + super(); + } + + public int compare(StringIntPair p1, StringIntPair p2) { + if (p1.getIntValue() < p2.getIntValue()) + return -1; + if (p1.getIntValue() > p2.getIntValue()) + return 1; + + return 0; + } + +} diff --git a/engine/src/main/java/org/archive/crawler/util/TopNSet.java b/engine/src/main/java/org/archive/crawler/util/TopNSet.java new file mode 100644 index 00000000..91c595de --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/util/TopNSet.java @@ -0,0 +1,95 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.util; + +import java.io.Serializable; +import java.util.HashMap; + +/** + * Counting Set which only remembers the 'top N' of all String values + * reported (with counts) to it. Assumes counts reported for a + * certain String key only ever increase. + * + * @contributor gojomo + */ +public class TopNSet implements Serializable { + + private static final long serialVersionUID = 1L; + + int maxsize; + HashMap set; + long smallestKnownValue; + String smallestKnownKey; + + public TopNSet(int size){ + maxsize = size; + set = new HashMap(size); + } + + public void update(String key, long value){ + if(set.containsKey(key)) { + // Update the value of an existing key + set.put(key,value); + // This may promote the key if it was the smallest + if(smallestKnownKey == null || smallestKnownKey.equals(key)){ + updateSmallest(); + } + } else if(set.size()smallestKnownValue){ + // Replace current smallest + set.remove(smallestKnownKey); + updateSmallest(); + set.put(key, value); + } // Else do nothing. + } + } + + private void updateSmallest(){ + // Need to scan through for new smallest value. + long oldSmallest = smallestKnownValue; + smallestKnownValue = Long.MAX_VALUE; + for(String k : set.keySet()){ + long v = set.get(k); + if(v next; + + public void setup(UURI source, UURI base, InputStream content, + Charset charset, ExtractErrorListener listener) { + setup(source, base, charSequenceFrom(content,charset), listener); + } + + /** + * @param source + * @param base + * @param content + * @param listener + */ + public void setup(UURI source, UURI base, CharSequence content, + ExtractErrorListener listener) { + this.source = source; + this.base = base; + this.extractErrorListener = listener; + this.sourceContent = content; + this.next = new LinkedList(); + } + + + /** + * Convenience method for when source and base are same. + * + * @param sourceandbase + * @param content + * @param listener + */ + public void setup(UURI sourceandbase, CharSequence content, + ExtractErrorListener listener) { + setup(sourceandbase, sourceandbase, content, listener); + } + + /* (non-Javadoc) + * @see org.archive.extractor.LinkExtractor#setup(org.archive.crawler.datamodel.UURI, java.io.InputStream, java.nio.charset.Charset) + */ + public void setup(UURI sourceandbase, InputStream content, Charset charset, + ExtractErrorListener listener) { + setup(sourceandbase,sourceandbase,content,charset,listener); + } + + /* (non-Javadoc) + * @see org.archive.extractor.LinkExtractor#nextLink() + */ + public Link nextLink() { + if(!hasNext()) { + throw new NoSuchElementException(); + } + // next will have been filled with at least one item + return (Link) next.removeFirst(); + } + + /** + * Discard all state. Another setup() is required to use again. + */ + public void reset() { + base = null; + source = null; + sourceContent = null; // TODO: discard other resources + } + + /* (non-Javadoc) + * @see java.util.Iterator#hasNext() + */ + public boolean hasNext() { + if (!next.isEmpty()) { + return true; + } + return findNextLink(); + } + + /** + * Scan to the next link(s), if any, loading it into the next buffer. + * + * @return true if any links are found/available, false otherwise + */ + abstract protected boolean findNextLink(); + + /* (non-Javadoc) + * @see java.util.Iterator#next() + */ + public Link next() { + return nextLink(); + } + + /* (non-Javadoc) + * @see java.util.Iterator#remove() + */ + public void remove() { + throw new UnsupportedOperationException(); + } + + /** + * @param content + * @param charset + * @return CharSequence obtained from stream in given charset + */ + protected CharSequence charSequenceFrom(InputStream content, Charset charset) { + // See if content InputStream can provide + if(content instanceof CharSequenceProvider) { + return ((CharSequenceProvider)content).getCharSequence(); + } + // otherwise, create one + return createCharSequenceFrom(content, charset); + } + + /** + * @param content + * @param charset + * @return CharSequence built over given stream in given charset + */ + protected CharSequence createCharSequenceFrom(InputStream content, Charset charset) { + // TODO: implement + return null; + // TODO: consider cleanup in reset() + } + + /** + * Convenience method to do default extraction. + * + * @param content + * @param source + * @param base + * @param collector + * @param extractErrorListener + */ + public static void extract(CharSequence content, UURI source, UURI base, + List collector, ExtractErrorListener extractErrorListener) { + // TODO: arrange for inheritance of prefs... eg when HTML includes JS + // includes HTML, have inner HTML follow robots, etc from outer + CharSequenceLinkExtractor extractor = newDefaultInstance(); + extractor.setup(source, base, content, extractErrorListener); + while (extractor.hasNext()) { + collector.add(extractor.nextLink()); + } + extractor.reset(); + } + + protected static CharSequenceLinkExtractor newDefaultInstance() { + // override in subclasses + return null; + } +} diff --git a/engine/src/main/java/org/archive/extractor/CharSequenceProvider.java b/engine/src/main/java/org/archive/extractor/CharSequenceProvider.java new file mode 100644 index 00000000..f5c94dab --- /dev/null +++ b/engine/src/main/java/org/archive/extractor/CharSequenceProvider.java @@ -0,0 +1,40 @@ +/* CharSequenceProvider +* +* $Id$ +* +* Created on Mar 17, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.extractor; + +/** + * Interface indicating an object can efficiently provide a + * (perhaps cached or simulated) CharSequence version of itself. + * + * @author gojomo + */ +public interface CharSequenceProvider { + + /** + * @return CharSequence linked/cached/implied by this object + */ + CharSequence getCharSequence(); + +} diff --git a/engine/src/main/java/org/archive/extractor/ExtractErrorListener.java b/engine/src/main/java/org/archive/extractor/ExtractErrorListener.java new file mode 100644 index 00000000..6011f6cc --- /dev/null +++ b/engine/src/main/java/org/archive/extractor/ExtractErrorListener.java @@ -0,0 +1,47 @@ +/* ExtractErrorListener +* +* $Id$ +* +* Created on Mar 17, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.extractor; + +import java.io.IOException; + +import org.archive.net.UURI; + +/** + * ExtractErrorListener receives exceptions that may need to be logged + * from inside a LinkExtractor, allowing the extraction to continue + * without raising an exception through hasNext()/next()/nextLink(). + * + * @author gojomo + */ +public interface ExtractErrorListener { + /** + * Callback to report an extraction error. + * + * @param ex + * @param source + * @param context + */ + public void noteExtractError(IOException ex, UURI source, CharSequence context); +} diff --git a/engine/src/main/java/org/archive/extractor/LinkExtractor.java b/engine/src/main/java/org/archive/extractor/LinkExtractor.java new file mode 100644 index 00000000..4aa26edc --- /dev/null +++ b/engine/src/main/java/org/archive/extractor/LinkExtractor.java @@ -0,0 +1,89 @@ +/* LinkExtractor +* +* $Id$ +* +* Created on Mar 16, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.extractor; + +import java.io.InputStream; +import java.nio.charset.Charset; +import java.util.Iterator; + +import org.archive.modules.extractor.Link; +import org.archive.net.UURI; + +/** + * LinkExtractor is a general interface for classes which, when given an + * InputStream and Charset, can scan for Links and return them via + * an Iterator interface. + * + * Implementors may in fact complete all extraction on the first + * hasNext(), then trickle Links out from an internal collection, + * depending on whether the link-extraction technique used is amenable + * to incremental scanning. + * + * ROUGH DRAFT IN PROGRESS / incomplete... untested... + * + * @author gojomo + */ +public interface LinkExtractor extends Iterator { + /** + * Setup the LinkExtractor to operate on the given stream and charset, + * considering the given contextURI as the initial 'base' URI for + * resolving relative URIs. + * + * May be called to 'reset' a LinkExtractor to start with new input. + * + * @param source source URI + * @param base base URI (usually the source URI) for URI derelativizing + * @param content input stream of content to scan for links + * @param charset Charset to consult to decode stream to characters + * @param listener ExtractErrorListener to notify, rather than raising + * exception through extraction loop + */ + public void setup(UURI source, UURI base, InputStream content, + Charset charset, ExtractErrorListener listener); + + /** + * Convenience version of above for common case where source and base are + * same. + * + * @param sourceandbase URI to use as source and base for derelativizing + * @param content input stream of content to scan for links + * @param charset Charset to consult to decode stream to characters + * @param listener ExtractErrorListener to notify, rather than raising + * exception through extraction loop + */ + public void setup(UURI sourceandbase, InputStream content, + Charset charset, ExtractErrorListener listener); + + /** + * Alternative to Iterator.next() which returns type Link. + * @return a discovered Link + */ + public Link nextLink(); + + /** + * Discard all state and release any used resources. + */ + public void reset(); +} diff --git a/engine/src/main/java/org/archive/extractor/RegexpCSSLinkExtractor.java b/engine/src/main/java/org/archive/extractor/RegexpCSSLinkExtractor.java new file mode 100644 index 00000000..07073b1e --- /dev/null +++ b/engine/src/main/java/org/archive/extractor/RegexpCSSLinkExtractor.java @@ -0,0 +1,117 @@ +/* + * ExtractorCSS + * + * $Id$ + * + * Created on Mar 29, 2005 + * + * Copyright (C) 2005 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +package org.archive.extractor; + +import java.util.regex.Matcher; + +import org.apache.commons.httpclient.URIException; +import org.archive.modules.extractor.Hop; +import org.archive.modules.extractor.Link; +import org.archive.modules.extractor.LinkContext; +import org.archive.net.UURIFactory; +import org.archive.util.DevUtils; +import org.archive.util.TextUtils; + +/** + * This extractor is parsing URIs from CSS type files. + * The format of a CSS URL value is 'url(' followed by optional white space + * followed by an optional single quote (') or double quote (") character + * followed by the URL itself followed by an optional single quote (') or + * double quote (") character followed by optional white space followed by ')'. + * Parentheses, commas, white space characters, single quotes (') and double + * quotes (") appearing in a URL must be escaped with a backslash: + * '\(', '\)', '\,'. Partial URLs are interpreted relative to the source of + * the style sheet, not relative to the document. + * Source: www.w3.org + * + * ROUGH DRAFT IN PROGRESS / incomplete... untested... major changes likely + * + * @author igor gojomo + * + **/ + +public class RegexpCSSLinkExtractor extends CharSequenceLinkExtractor { + + // private static Logger logger = + // Logger.getLogger(RegexpCSSLinkExtractor.class.getName()); + + private static String ESCAPED_AMP = "&"; + // CSS escapes: "Parentheses, commas, whitespace characters, single + // quotes (') and double quotes (") appearing in a URL must be + // escaped with a backslash" + static final String CSS_BACKSLASH_ESCAPE = "\\\\([,'\"\\(\\)\\s])"; + + protected Matcher uris; + + /** + * CSS URL extractor pattern. + * + * This pattern extracts URIs for CSS files + **/ + static final String CSS_URI_EXTRACTOR = + "(?:@import (?:url[(]|)|url[(])\\s*([\\\"\']?)([^\\\"\'].*?)\\1\\s*[);]"; + + protected boolean findNextLink() { + if (uris == null) { + uris = TextUtils.getMatcher(CSS_URI_EXTRACTOR, sourceContent); + // NOTE: this matcher can't be recycled in this method because + // it is reused on rentry + } + String cssUri; + try { + while (uris.find()) { + cssUri = uris.group(2); + // TODO: Escape more HTML Entities. + cssUri = TextUtils.replaceAll(ESCAPED_AMP, cssUri, "&"); + // Remove backslashes when used as escape character in CSS URL + cssUri = TextUtils.replaceAll(CSS_BACKSLASH_ESCAPE, cssUri, "$1"); + // TODO: handle relative URIs? + try { + Link link = new Link(source, UURIFactory.getInstance(base, + cssUri), LinkContext.EMBED_MISC, Hop.EMBED); + next.addLast(link); + } catch (URIException e) { + extractErrorListener.noteExtractError(e, source, cssUri); + } + return true; + } + } catch (StackOverflowError e) { + DevUtils.warnHandle(e, "RegexpCSSLinkExtractor StackOverflowError"); + } + return false; + } + + public void reset() { + super.reset(); + TextUtils.recycleMatcher(uris); + uris = null; + } + + protected static CharSequenceLinkExtractor newDefaultInstance() { + return new RegexpCSSLinkExtractor(); + } +} diff --git a/engine/src/main/java/org/archive/extractor/RegexpHTMLLinkExtractor.java b/engine/src/main/java/org/archive/extractor/RegexpHTMLLinkExtractor.java new file mode 100644 index 00000000..dfdd4e4c --- /dev/null +++ b/engine/src/main/java/org/archive/extractor/RegexpHTMLLinkExtractor.java @@ -0,0 +1,466 @@ +/* Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * SimpleHTMLExtractor.java + * Created on Jun 5, 2003 + * + * $Header$ + */ +package org.archive.extractor; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; + +import org.apache.commons.httpclient.URIException; +import org.archive.modules.extractor.HTMLLinkContext; +import org.archive.modules.extractor.Hop; +import org.archive.modules.extractor.Link; +import org.archive.modules.extractor.LinkContext; +import org.archive.net.UURI; +import org.archive.net.UURIFactory; +import org.archive.util.DevUtils; +import org.archive.util.TextUtils; + + +/** + * Basic link-extraction, from an HTML content-body, + * using regular expressions. + * + * ROUGH DRAFT IN PROGRESS / incomplete... untested... + * + * @author gojomo + */ +public class RegexpHTMLLinkExtractor extends CharSequenceLinkExtractor { + private static Logger logger = + Logger.getLogger(RegexpHTMLLinkExtractor.class.getName()); + + boolean honorRobots = true; + boolean extractInlineCss = true; + boolean extractInlineJs = true; + + protected LinkedList next = new LinkedList(); + protected Matcher tags; + + /* (non-Javadoc) + * @see org.archive.extractor.CharSequenceLinkExtractor#findNextLink() + */ + protected boolean findNextLink() { + if (tags == null) { + tags = TextUtils.getMatcher(RELEVANT_TAG_EXTRACTOR, sourceContent); + } + while(tags.find()) { + if(Thread.interrupted()){ + // TODO: throw an exception, perhaps, rather than just clear & break? + break; + } + if (tags.start(8) > 0) { + // comment match + // for now do nothing + } else if (tags.start(7) > 0) { + // match + int start = tags.start(5); + int end = tags.end(5); + processMeta(sourceContent.subSequence(start, end)); + } else if (tags.start(5) > 0) { + // generic match + int start5 = tags.start(5); + int end5 = tags.end(5); + int start6 = tags.start(6); + int end6 = tags.end(6); + processGeneralTag(sourceContent.subSequence(start6, end6), + sourceContent.subSequence(start5, end5)); + } else if (tags.start(1) > 0) { + // + + +

      Simple Javascript Extraction Self Test

      +

      This page is meant to test the ability of the parser to get links + out of javascript.

      + +

      Links: +

      +

      + + + diff --git a/engine/src/webapps/selftest/SimpleJavascriptExtraction/jscriptOpenWindow.html b/engine/src/webapps/selftest/SimpleJavascriptExtraction/jscriptOpenWindow.html new file mode 100644 index 00000000..98083a1a --- /dev/null +++ b/engine/src/webapps/selftest/SimpleJavascriptExtraction/jscriptOpenWindow.html @@ -0,0 +1,9 @@ + + +javascript popup window + + + +This page should be found by parsing a pretty easy relative link in a javascript window.open call. + + \ No newline at end of file diff --git a/engine/src/webapps/selftest/SimpleJavascriptExtraction/jscriptOpenWindowArg.html b/engine/src/webapps/selftest/SimpleJavascriptExtraction/jscriptOpenWindowArg.html new file mode 100644 index 00000000..d6ac1143 --- /dev/null +++ b/engine/src/webapps/selftest/SimpleJavascriptExtraction/jscriptOpenWindowArg.html @@ -0,0 +1,10 @@ + + +javascript popup window (function param test) + + + +This page should be found by parsing a relative link that has been passed to window.open as a function argument +(e.g. function('thispage.html'). + + \ No newline at end of file diff --git a/engine/src/webapps/selftest/SpacesInHrefPath/index.html b/engine/src/webapps/selftest/SpacesInHrefPath/index.html new file mode 100644 index 00000000..5315d8cc --- /dev/null +++ b/engine/src/webapps/selftest/SpacesInHrefPath/index.html @@ -0,0 +1,10 @@ + + + Spaces In HREF Self Test + + + +

      Spaces In HREF Self Test

      +

      spaces in path.

      + + diff --git a/engine/src/webapps/selftest/SpacesInHrefPath/spaces in path.html b/engine/src/webapps/selftest/SpacesInHrefPath/spaces in path.html new file mode 100644 index 00000000..21b606a0 --- /dev/null +++ b/engine/src/webapps/selftest/SpacesInHrefPath/spaces in path.html @@ -0,0 +1,5 @@ + + +This file has spaces in its name. Isn't that special? + + diff --git a/engine/src/webapps/selftest/TrickyRelativeURIs/anothersub/reluptarget.html b/engine/src/webapps/selftest/TrickyRelativeURIs/anothersub/reluptarget.html new file mode 100644 index 00000000..da3fdcf0 --- /dev/null +++ b/engine/src/webapps/selftest/TrickyRelativeURIs/anothersub/reluptarget.html @@ -0,0 +1 @@ +This page should be found by parsing a relative URI. diff --git a/engine/src/webapps/selftest/TrickyRelativeURIs/index.html b/engine/src/webapps/selftest/TrickyRelativeURIs/index.html new file mode 100644 index 00000000..b4311de7 --- /dev/null +++ b/engine/src/webapps/selftest/TrickyRelativeURIs/index.html @@ -0,0 +1,17 @@ + + +Tricky Relative URIs Self Test + + + +

      Tricky Relative URIs Self Test

      +

      This test test being able to follow awkward relatives paths.

      +

      relative-up

      + +

      tricky + relative-up +

      + + diff --git a/engine/src/webapps/selftest/TrickyRelativeURIs/reluptricky.html b/engine/src/webapps/selftest/TrickyRelativeURIs/reluptricky.html new file mode 100644 index 00000000..ce3bbeed --- /dev/null +++ b/engine/src/webapps/selftest/TrickyRelativeURIs/reluptricky.html @@ -0,0 +1 @@ +This is a relatively tricky relative URI. diff --git a/engine/src/webapps/selftest/WEB-INF/web.xml b/engine/src/webapps/selftest/WEB-INF/web.xml new file mode 100644 index 00000000..0e87acb3 --- /dev/null +++ b/engine/src/webapps/selftest/WEB-INF/web.xml @@ -0,0 +1,33 @@ + + + + + Selftest + Suite of tests for a web crawler. + + + Basic auth constraint. + + Selftest Basic Auth Test + /Auth/basic/* + + + selftest + + + + + + BASIC + selftest + + + + Role used to get into the area protected by BASIC auth. + + selftest + + + diff --git a/engine/src/webapps/selftest/index.jsp b/engine/src/webapps/selftest/index.jsp new file mode 100644 index 00000000..61a08568 --- /dev/null +++ b/engine/src/webapps/selftest/index.jsp @@ -0,0 +1,81 @@ +<%@ page contentType="text/html; charset=ISO-8859-1" %> +<%@ page import="java.io.File" %> +<%@ page import="java.util.ArrayList" %> +<%@ page import="java.util.Iterator" %> +<%@ page import="org.archive.crawler.Heritrix" %> +<% + // This code looks for all subdirs -- each test occupies its own subdir. + // Assumption is that the war file has been extracted else this technique + // will fail. We exclude CVS and WEB-INF dirs as well as all selftests + // not yet implemented. + File cwd = new File(pageContext.getServletContext(). + getRealPath(File.separator)); + ArrayList dirs = new ArrayList(); + File [] files = cwd.listFiles(); + if (files != null) { + for (int i = 0; i < files.length; i++) { + if (files[i].isDirectory() && + !files[i].getName().equals("TrickyRelativeURIs") && + !files[i].getName().equals("SpacesInHrefPath") && + !files[i].getName().equals("SimpleJavascriptExtraction") && + !files[i].getName().equals("RobotsExclusion") && + !files[i].getName().equals("Refresh") && + !files[i].getName().equals("FormTagExtraction") && + !files[i].getName().equals("SimpleDocumentTypes") && + !files[i].getName().equals("WEB-INF") && + !files[i].getName().equals("CVS")) { + dirs.add(files[i].getName()); + } + } + } + Iterator dirsIterator = dirs.iterator(); +%> + + + + + + + Heritrix Crawler Garden Home Page + + + +

      Heritrix Crawler Garden Home Page

      + +

      This is the home page for the serverside of the Heritrix crawler + integration self test. The clientside of + the integration self test can be found in the + org.archive.crawler.selftest package. See the javadoc for + this package for more on the integration self test including how to add + new tests.

      + +

      The integration self test is run from the command line. This + will start a crawler that will meander here, + in this selftest webapp. Code on the + client validates successful crawler traversal of all tests.

      + +

      Below are the tests to run. Each test is totally contained in a + subdirectory named for the test. This page lists all test + subdirectories. The crawler in integration self test mode is + pointed at this page. It runs the tests in no particular order.

      + +

      Integration Tests

      +

      +

        + <% + String dir = null; + while (dirsIterator.hasNext()) { + dir = (String)dirsIterator.next(); + %> +
      • <%=dir%>
        • + <% + } + %> +
      +

      +
      + Heritrix version <%=Heritrix.getVersion()%>, $Id$ + + + diff --git a/engine/src/webapps/selftest/robots.txt b/engine/src/webapps/selftest/robots.txt new file mode 100644 index 00000000..5720d848 --- /dev/null +++ b/engine/src/webapps/selftest/robots.txt @@ -0,0 +1,4 @@ +# This robots.txt is used by the RobotsExclusion heritrix integration self test. +User-agent: heritrix_selftest(+http://localhost.localdomain) +Disallow: /RobotsExclusion/excluded.html +Disallow: /RobotsExclusion/excluded/ diff --git a/engine/src/xsd/arc/1.0/arc.html b/engine/src/xsd/arc/1.0/arc.html new file mode 100644 index 00000000..bb0484e3 --- /dev/null +++ b/engine/src/xsd/arc/1.0/arc.html @@ -0,0 +1,1762 @@ + + + + + + Internet Archive ARC file metadata XML Schema + + + + + +

      + Internet Archive ARC file metadata XML Schema +

      + + + + +

      Table of Contents

      + + + +
      + top +
      +
      +

      + Schema Document Properties +

      + + + + + + + + + + + + + + + + + +
      + Target Namespace + + http://archive.org/arc/1.0/ +
      Element and Attribute Namespaces +
        +
      • Global element and attribute declarations belong to this schema's target namespace.
        • +
      • By default, local element declarations belong to this schema's target namespace.
        • +
      • By default, local attribute declarations have no namespace.
        • +
      +
      Schema Composition +
        +
      • This schema imports schema(s) from the following namespace(s):
          +
        • + http://www.w3.org/XML/1998/namespace (at http://www.w3.org/2001/03/xml.xsd)
          • +
        • + http://purl.org/dc/elements/1.1/ (at http://dublincore.org/schemas/xmls/qdc/2003/04/02/dc.xsd)
          • +
        • + http://purl.org/dc/terms/ (at http://dublincore.org/schemas/xmls/qdc/2003/04/02/dcterms.xsd)
          • +
        +
        • +
      +
      Documentation + Internet Archive ARC file Dublin Core element refinements. + + XML Schema for http://archive.org/arc/1.0/ namespace. + + This schema adds Dublin Core element refinements for Internet + Achive ARC file metadata. It also adds new elements + that subclass the 'dc:any' type and the 'dc:metadata' + element container. + + $Id$ + +
      +

      Declared Namespaces

      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      PrefixNamespace
      + Default namespace + + http://archive.org/arc/1.0/ +
      + xml + http://www.w3.org/XML/1998/namespace
      + arc + + http://archive.org/arc/1.0/ +
      + xs + http://www.w3.org/2001/XMLSchema
      + dc + http://purl.org/dc/elements/1.1/
      + dcterms + http://purl.org/dc/terms/
      + + + + + + + +
      + Schema Component Representation
      +
      +
      <xs:schema + targetNamespace="http://archive.org/arc/1.0/" elementFormDefault="qualified" attributeFormDefault="unqualified">
      <xs:import + namespace="http://www.w3.org/XML/1998/namespace" schemaLocation="http://www.w3.org/2001/03/xml.xsd"/>
      +
      <xs:import + namespace="http://purl.org/dc/elements/1.1/" schemaLocation="http://dublincore.org/schemas/xmls/qdc/2003/04/02/dc.xsd"/>
      +
      <xs:import + namespace="http://purl.org/dc/terms/" schemaLocation="http://dublincore.org/schemas/xmls/qdc/2003/04/02/dcterms.xsd"/>
      +
      ...
      </xs:schema>
      +
      + +
      +
      + top +
      +
      +

      + Global Declarations +

      +

      Element: arcmetadata +

      + + + + + + + + + + + + + + + + + + + + + +
      Namearcmetadata
      Type + + dcterms:elementOrRefinementContainer + +
      + Nillable + no
      + Abstract + no
      Documentation Root container for ARC metadata. + Can hold any 'dc:any' element. (All ARC elements are subclasses of + the 'dc:any' type).
      + + + + + + + +
      + XML Instance Representation
      +
      +
      <arcmetadata> + dcterms:elementOrRefinementContainer + </arcmetadata>
      +
      + +
      + + + + + + + +
      + Schema Component Representation
      +
      +
      <xs:element + name="arcmetadata" type=" + + dcterms:elementOrRefinementContainer + + "/>
      +
      + +
      +
      + top +
      +
      +

      Element: hostname +

      + + + + +
      +
        +
      • + This element can be used wherever the following element is referenced: + +
        • +
      +
      + + + + + + + + + + + + + + + + + + + + + +
      Namehostname
      TypeanyType
      + Nillable + no
      + Abstract + no
      Documentation Element-refinement for 'dc:creator'. + The hostname of the machine that created this resource. Example + 'debord.archive.org'.
      + + + + + + + +
      + XML Instance Representation
      +
      +
      <hostname> ... </hostname>
      +
      + +
      + + + + + + + +
      + Schema Component Representation
      +
      +
      <xs:element + name="hostname" substitutionGroup="dc:creator"/>
      +
      + +
      +
      + top +
      +
      +

      Element: http-header-from +

      + + + + +
      +
        +
      • + This element can be used wherever the following element is referenced: + +
        • +
      +
      + + + + + + + + + + + + + + + + + + + + + +
      Namehttp-header-from
      TypeanyType
      + Nillable + no
      + Abstract + no
      DocumentationThe HTTP 'From' header sent by the + crawler along with each request. Example: + 'webmaster@localhost.localdomain'. Note that this value will be + removed from metadata when ARC files record the request sent + (Currently the response only is recorded in ARCs).
      + + + + + + + +
      + XML Instance Representation
      +
      +
      <http-header-from> ... </http-header-from>
      +
      + +
      + + + + + + + +
      + Schema Component Representation
      +
      +
      <xs:element + name="http-header-from" substitutionGroup="dc:any"/>
      +
      + +
      +
      + top +
      +
      +

      Element: http-header-user-agent +

      + + + + +
      +
        +
      • + This element can be used wherever the following element is referenced: + +
        • +
      +
      + + + + + + + + + + + + + + + + + + + + + +
      Namehttp-header-user-agent
      TypeanyType
      + Nillable + no
      + Abstract + no
      Documentation The HTTP 'user-agent' header sent + by the crawler along with each request. Example: + 'os-heritrix/0.7.0 (+http://crawler.archive.org)'. Note that + this value will be removed from metadata when ARC files record + the request sent (Currently the response only is recorded in + ARCs).
      + + + + + + + +
      + XML Instance Representation
      +
      +
      <http-header-user-agent> ... </http-header-user-agent>
      +
      + +
      + + + + + + + +
      + Schema Component Representation
      +
      +
      <xs:element + name="http-header-user-agent" substitutionGroup="dc:any"/>
      +
      + +
      +
      + top +
      +
      +

      Element: ip +

      + + + + +
      +
        +
      • + This element can be used wherever the following element is referenced: + +
        • +
      +
      + + + + + + + + + + + + + + + + + + + + + +
      Nameip
      TypeanyType
      + Nillable + no
      + Abstract + no
      Documentation Element-refinement for 'dc:creator'. + The IP address of the machine that created this resource. + Example '123.1.2.3'.
      + + + + + + + +
      + XML Instance Representation
      +
      +
      <ip> ... </ip>
      +
      + +
      + + + + + + + +
      + Schema Component Representation
      +
      +
      <xs:element + name="ip" substitutionGroup="dc:creator"/>
      +
      + +
      +
      + top +
      +
      +

      Element: operator +

      + + + + +
      +
        +
      • + This element can be used wherever the following element is referenced: + +
        • +
      +
      + + + + + + + + + + + + + + + + + + + + + +
      Nameoperator
      TypeanyType
      + Nillable + no
      + Abstract + no
      Documentation Element-refinement for 'dc:creator'. + Contact name and information for the operator who created this resource. + Example: 'stack at archive dot org'.
      + + + + + + + +
      + XML Instance Representation
      +
      +
      <operator> ... </operator>
      +
      + +
      + + + + + + + +
      + Schema Component Representation
      +
      +
      <xs:element + name="operator" substitutionGroup="dc:creator"/>
      +
      + +
      +
      + top +
      +
      +

      Element: robots +

      + + + + +
      +
        +
      • + This element can be used wherever the following element is referenced: + +
        • +
      +
      + + + + + + + + + + + + + + + + + + + + + +
      Namerobots
      TypeanyType
      + Nillable + no
      + Abstract + no
      Documentation + Element-refinement for 'dc:coverage'. + The robots policy followed by the + crawler harvesting this resource. Example: 'classic'. +
      + + + + + + + +
      + XML Instance Representation
      +
      +
      <robots> ... </robots>
      +
      + +
      + + + + + + + +
      + Schema Component Representation
      +
      +
      <xs:element + name="robots" substitutionGroup="dc:coverage"/>
      +
      + +
      +
      + top +
      +
      +

      Element: software +

      + + + + +
      +
        +
      • + This element can be used wherever the following element is referenced: + +
        • +
      +
      + + + + + + + + + + + + + + + + + + + + + +
      Namesoftware
      TypeanyType
      + Nillable + no
      + Abstract + no
      Documentation Element-refinement for 'dc:creator'. + The software and software version used creating this resource. + Example: 'heritrix 0.7.1 http://crawler.archive.org'.
      + + + + + + + +
      + XML Instance Representation
      +
      +
      <software> ... </software>
      +
      + +
      + + + + + + + +
      + Schema Component Representation
      +
      +
      <xs:element + name="software" substitutionGroup="dc:creator"/>
      +
      + +
      +
      + top +
      +
      +
      +

      + Legend +

      + + + + + + + + + +
      + Complex Type: + + AusAddress +
      + Schema Component Type + + Schema Component Name +
      + + + + + + + + + +
      Super-types: + Address < AusAddress (by extension)
      Sub-types: +
        +
      • + QLDAddress (by restriction)
        • +
      +
      +
      If this schema component is a type definition, its type hierarchy is shown in a gray-bordered box.
      + + + + + + + + + +
      NameAusAddress
      + Abstract + no
      +
      The table above displays the properties of this schema component.
      + + + + + + + +
      XML Instance Representation
      +
      + <... + + country="Australia" + >
      + <unitNo> string </unitNo> [0..1] + +
      + <houseNo> string </houseNo> [1] + +
      + <street> string </street> [1] + +
      + Start Choice + [1] + +
      + <city> string </city> [1] + +
      + <town> string </town> [1] + +
      + End Choice +
      + + <state> AusStates </state> [1] + +
      + <postcode> string <<pattern = [1-9][0-9]{3}>> </postcode> [1] + ? + +
      +       + </...> +
      +
      +
      +
      +

      The XML Instance Representation table above shows the schema component's content as an XML instance.

      +
        +
      • The minimum and maximum occurrence of elements and attributes are provided in square brackets, e.g. [0..1].
        • +
      • Model group information are shown in gray, e.g. Start Choice ... End Choice.
        • +
      • For type derivations, the elements and attributes that have been added to or changed from the base type's content are shown in bold.
        • +
      • If an element/attribute has a fixed value, the fixed value is shown in green, e.g. country="Australia".
        • +
      • Otherwise, the type of the element/attribute is displayed.
          +
        • If the element/attribute's type is in the schema, a link is provided to it.
          • +
        • For local simple type definitions, the constraints are displayed in angle brackets, e.g. <<pattern = [1-9][0-9]{3}>>.
          • +
        +
        • +
      • If a local element/attribute has documentation, it will be displayed in a window that pops up when the question mark inside the attribute or next to the element is clicked, e.g. <postcode>.
        • +
      +
      + + + + + + + +
      Schema Component Representation
      +
      + <complexType + name="AusAddress"> +
      + <complexContent> +
      + <extension + base=" + Address + "> +
      + <sequence> +
      + <element + name="state" type=" + AusStates + "/> +
      + <element + name="postcode"> +
      + <simpleType> +
      + <restriction + base=" + string + "> +
      + <pattern + value="[1-9][0-9]{3}"/> +
      + </restriction> +
      + </simpleType> +
      + </element> +
      + </sequence> +
      + <attribute + name="country" type=" + string + " fixed="Australia"/> +
      + </extension> +
      + </complexContent> +
      + </complexType> +
      +
      +
      +
      The Schema Component Representation table above displays the underlying XML representation of the schema component. (Annotations are not shown.)
      +
      + top +
      +
      +
      +
      +

      + Glossary +

      +

      + + Abstract + (Applies to complex type definitions and element declarations). An abstract element or complex type cannot used to validate an element instance. If there is a reference to an abstract element, only element declarations that can substitute the abstract element can be used to validate the instance. For references to abstract type definitions, only derived types can be used.

      +

      + + All Model Group + Child elements can be provided in any order in instances. See: http://www.w3.org/TR/xmlschema-1/#element-all.

      +

      + + Choice Model Group + + Only one from the list of child elements and model groups can be provided in instances. See: http://www.w3.org/TR/xmlschema-1/#element-choice.

      +

      + + Collapse Whitespace Policy + Replace tab, line feed, and carriage return characters with space character (Unicode character 32). Then, collapse contiguous sequences of space characters into single space character, and remove leading and trailing space characters.

      +

      + + Disallowed Substitutions + (Applies to element declarations). If substitution is specified, then substitution group members cannot be used in place of the given element declaration to validate element instances. If derivation methods, e.g. extension, restriction, are specified, then the given element declaration will not validate element instances that have types derived from the element declaration's type using the specified derivation methods. Normally, element instances can override their declaration's type by specifying an xsi:type attribute.

      +

      + + Key Constraint + Like Uniqueness Constraint, but additionally requires that the specified value(s) must be provided. See: http://www.w3.org/TR/xmlschema-1/#cIdentity-constraint_Definitions.

      +

      + + Key Reference Constraint + Ensures that the specified value(s) must match value(s) from a Key Constraint or Uniqueness Constraint. See: http://www.w3.org/TR/xmlschema-1/#cIdentity-constraint_Definitions.

      +

      + + Model Group + Groups together element content, specifying the order in which the element content can occur and the number of times the group of element content may be repeated. See: http://www.w3.org/TR/xmlschema-1/#Model_Groups.

      +

      + + Nillable + (Applies to element declarations). If an element declaration is nillable, instances can use the xsi:nil attribute. The xsi:nil attribute is the boolean attribute, nil, from the http://www.w3.org/2001/XMLSchema-instance namespace. If an element instance has an xsi:nil attribute set to true, it can be left empty, even though its element declaration may have required content.

      +

      + + Notation + A notation is used to identify the format of a piece of data. Values of elements and attributes that are of type, NOTATION, must come from the names of declared notations. See: http://www.w3.org/TR/xmlschema-1/#cNotation_Declarations.

      +

      + + Preserve Whitespace Policy + Preserve whitespaces exactly as they appear in instances.

      +

      + + Prohibited Derivations + (Applies to type definitions). Derivation methods that cannot be used to create sub-types from a given type definition.

      +

      + + Prohibited Substitutions + (Applies to complex type definitions). Prevents sub-types that have been derived using the specified derivation methods from validating element instances in place of the given type definition.

      +

      + + Replace Whitespace Policy + Replace tab, line feed, and carriage return characters with space character (Unicode character 32).

      +

      + + Sequence Model Group + Child elements and model groups must be provided in the specified order in instances. See: http://www.w3.org/TR/xmlschema-1/#element-sequence.

      +

      + + Substitution Group + Elements that are members of a substitution group can be used wherever the head element of the substitution group is referenced.

      +

      + + Substitution Group Exclusions + (Applies to element declarations). Prohibits element declarations from nominating themselves as being able to substitute a given element declaration, if they have types that are derived from the original element's type using the specified derivation methods.

      +

      + + Target Namespace + The target namespace identifies the namespace that components in this schema belongs to. If no target namespace is provided, then the schema components do not belong to any namespace.

      +

      + + Uniqueness Constraint + Ensures uniqueness of an element/attribute value, or a combination of values, within a specified scope. See: http://www.w3.org/TR/xmlschema-1/#cIdentity-constraint_Definitions.

      +
      + top +
      +
      +
      +

      Generated by xs3p. Last modified: +

      + + \ No newline at end of file diff --git a/engine/src/xsd/arc/1.0/arc.xsd b/engine/src/xsd/arc/1.0/arc.xsd new file mode 100644 index 00000000..60a25991 --- /dev/null +++ b/engine/src/xsd/arc/1.0/arc.xsd @@ -0,0 +1,92 @@ + + + + + Internet Archive ARC file Dublin Core element refinements. + + XML Schema for http://archive.org/arc/1.0/ namespace. + + This schema adds Dublin Core element refinements for Internet + Achive ARC file metadata. It also adds new elements + that subclass the 'dc:any' type and the 'dc:metadata' + element container. + + $Id$ + + + + + + + + + + Element-refinement for 'dc:coverage'. + The robots policy followed by the + crawler harvesting this resource. Example: 'classic'. + + + + + + The HTTP 'From' header sent by the + crawler along with each request. Example: + 'webmaster@localhost.localdomain'. Note that this value will be + removed from metadata when ARC files record the request sent + (Currently the response only is recorded in ARCs). + + + + + The HTTP 'user-agent' header sent + by the crawler along with each request. Example: + 'os-heritrix/0.7.0 (+http://crawler.archive.org)'. Note that + this value will be removed from metadata when ARC files record + the request sent (Currently the response only is recorded in + ARCs). + + + + + Element-refinement for 'dc:creator'. + The software and software version used creating this resource. + Example: 'heritrix 0.7.1 http://crawler.archive.org'. + + + + + Element-refinement for 'dc:creator'. + The hostname of the machine that created this resource. Example + 'debord.archive.org'. + + + + + Element-refinement for 'dc:creator'. + The IP address of the machine that created this resource. + Example '123.1.2.3'. + + + + + Element-refinement for 'dc:creator'. + Contact name and information for the operator who created this resource. + Example: 'stack at archive dot org'. + + + + + Root container for ARC metadata. + Can hold any 'dc:any' element. (All ARC elements are subclasses of + the 'dc:any' type). + + + diff --git a/engine/src/xsd/arc/1.0/example.xml b/engine/src/xsd/arc/1.0/example.xml new file mode 100644 index 00000000..370e07c7 --- /dev/null +++ b/engine/src/xsd/arc/1.0/example.xml @@ -0,0 +1,28 @@ + + + Internet Archive + Election 2004 May Crawl + E04May-20040519021415-00023-debord.arc.gz + 2004-04-29T23:50:02 + + E04May + Library of Congress + crawl04.archive.org + 123.1.2.3 + stack at archive dot org + Heritrix 0.7.1 http://crawler.archive.org + classic + stack@crawler.archive.org + os-heritrix/0.7.1 + (+http://crawler.archive.org) + + ARC file version 1.1 + http://www.archive.org/web/researcher/ArcFileFormat.php + + diff --git a/engine/testdata/selftest/BackgroundImageExtractionSelfTestCase/htdocs/example-background-image.jpeg b/engine/testdata/selftest/BackgroundImageExtractionSelfTestCase/htdocs/example-background-image.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..fed1c1d626cdb0a7953bf9e2b258910992e36cfb GIT binary patch literal 6413 zcmbW3c{E$yzyCuhQj5|=Q8Ps-ZHRepYls>XW5n#Cp*-fIG_=Mv#+Vvvj9FUqP*t?% zQWa{ds+uWE(H6x`pXa-NzqRgP_uhThIs2T?{_MTpulN3}z0TRx**xHap&mjHKt~4v z(9tfy*+;-lzjntYbTZQY(*XZzbo4+5#&b-}ETHqWgvJX1dO9GGo&m_n$iP6$j-ZVL z7&sU?g<(49E}AggLG3{e)AR@OGQXjiwt+&%7KJp%%Rg71ZdhCPaoiH(a-z^4(@GcvQXpOA`* z$t9&_Fw(u7^IGkf0&q@nx6Txu(-6mvbwguvHN{*|KP{Z zUx!D3xaa`Df3axKf5HA67YB`to`C_#!1RZUj{Y9)268Yk3d7EE>X{4 z!DB4A{`M2IeG#p9I-Gk3h_UUlAE=o{E%?3F&J}I{!hTj z+|R~cvPTP?UFJ*6{Wk*Y5r_UF_gtcDA@`^fP%W`8MDTB<%hOI;rW)Agar{3$ZN_E1 z5o#BKmle(?i{#@CWr2POd@vMdbZ#N-DITUByFdhQ7@0BBG^B$tR!WLB+0VlC$8n9i zS$U_ncVqf{N6rA2fst0OtiRHwS0lDsmy0Q5K51sswH+bIAf@f-mD? zCNyI(!8~}FX^*&Mi;WMI_QG98lmE|4(HG}z@;@B^iHJjIt$6;UPx=qhR55s4nw4C} zp{o`-V}z>xqq~I+-}qN{5O|wLQRYI!1(8dN9!Ji=ZvS!TQfC>R9(3y2{hTRnY5C^~ z`i&h26f$&h8NcgaU2W68(c+2Le6tm61b zENFCyk1S@6&+-;VhreDH=@FN@0fp)IxTqst$RZ?N%Yi%paL`oeu>Nbm8vYFofqwvn znJm-{AvsKYgu&ZfJO~ro#z11Jynf*HZD|m>3~BCrnHHpNhOW$&OBd498_a#-G(pj9+M|2#SX-;aE6UdDjizMVk7%4BVq5L5SCq+l6Z$-^!v4`)PH10rij4Y| zo2iCk&e&j!F6c)o%v*EmEwZ$gBFfOLRD!b=jYZcWNyrL=EKd3cYNHAo3o*Rak^&)% zl>AE&kHMQ_-;TQeYz}ld}dcj?k!?B)L$=#UhD!P2m z&&NWi>Ciia%O5TdMEExgi_`||jMp3!mz4QhW4t@2AdW3@d)} zR`p+^Hr*~I>iQ^hklY(m&(K(Ub?w;i%BGt87I~)pQha?ylA#uF`0r$ER4T^CTpxY1 z42|*J>oHRgBD+g)7NQelY~9MIy(~qQI9q9`R^ZoCb%Bbc z(hRx+Mxt75rp%F&r4)n4&ZT?h$>z4IcSaHv1Pe|xYKrO_3iB^)=>I+-myq#(A9&M} z_rG4NIs>dJl<+>J&^t^2322?0U_b8NfiA{ zjbaGvg2&1WHFw{UH}x_Pw@{BAR4vz>xCg4}ud7)btHW0-R!2B0QYKoqESDbfSA`Zc zHCmBoEk3+4$t$_Z5$rSou}YJgB=K27Ia|k*)=XBsN~PK;j;T`h6fYj$28+MSvn{hA z=Rwm3ah&wfXp3qoc`LH}f*!Us4;xD{h~p^DrupO9n&5X?PqL^&?k;zQXnboaz8RbN zMu9UUr5Zl{w6zD7DHliDa5W%l`q|8oit;n-6ZMB(3I*=W_QS^v`$*0T<4-aospg~` zG(*AZ7mbJ2@UKUWhk0lx!$)+g=M@E-#-%V%u>so`%J14L@^N3$SR0b}RMX@&0zSN% z=%u_Cr|vP6Yd>BMvMnx3!Ie#&0bZT~jzBhJVyW5PmNz8()t8p?cgn9Un$?^ELeBvE zyhPK8X4o^ESi+8K#3E*E&CV|8y|@A8cF2)cL>1V{Vt(Q!rw?CLzPY829;)8sNjy=) zvI1s|2`;myqVR-3^?6Z+i!!&(Yb&aWNyT~*LlItbmR*$lX)4vNZIeTmbvXw6ca&TR zk1gHMt-_UQyofx4g04OUXc?zqKkWI$m_y;#2T|UHM9(K|qAaCE!`@XjpuMJWwmX5$>O^fN6xcvm@``Icd-E1)=hSQV=Xz zmBsZpR>WYpo{N?F&~HrhqKL_|Z@Ocn>*0(`I$P_^EC_;&K^7&1^wDG+X zL&PRp#@yi8f;dyCB>mNL_2tLQ^Jf5-qF0)7Ryw@My9n|p3vmt4P#@Qj^r&}Ct`=_T zxW+@?-Q1je8Xz;TLe|*I0Te=V|L^_}9_hHCGr%_z`Q0xO9;vqR=aQ>SxZ{yjp3w?B%UA-c$H@SxVMkvmqeyX@k|ZhGr(J$sGlfXl~SVJeePhn zA9dS4{GNQi$@wNyiW3W47xTNVVDK?xvnbhyM3#e4^1`)G$wT9!s27fho~GkKOQ^K2 z2^8JyL(WIOb)2moc+vsNl(7mqJ>+D6(5RP|FXnh7bm;}DqIz$yr|($2RJHMyYV#N* zPS>>d5!8yLU}wv7=%V~nfoDQ&gri(wrT$K(Md4HyI%1gFr>WaPS7y)2HR;3Ey{q@` zjHa0tII;6rX_#&|;TrR)Pty#qBOruj@isVTeL<+vb7 z{X5-hqmT3^j=554lskspQ|BIi?kM@R37XoW@VcVcyp_5#R!c=Ya5=$}OM+B`$o^8T za<>=WR{Oq>~SDbi|avENYh6i9ItjD5(!`eK9vM4U5av;*0lY9XK=)d(K z{~qb9f@FY?Z%jO7b%t~p(CpCAj{lJ)cgwleUai3v$MH!jl1uiS3cB+2O55qCEVi(FA zo?}{~CTyQ_Nc+-H&!0tSKW`MDVGO>xEb)^o8LHLNs5qlI2I&uPI0NK)9U?=$Xv!zQ zS2T?_&qa>My>;BawtSnWdb?HQ#0+T$l6c=@6H+ZL{OI%^RpriNSIQ*g<_ z%|Tp3+h}ZaqX(Dl(S^CgSGTI>iMr<9%HO;$FbkND4n-7|8u2X;7fS_kn=P` z@XbVvaTxSw+tswek4?*w!pfoohy5!VA>Ti=?9`1G6a94#qd>E~F(1{v%S=k#oVc6j zPBZM2^ilhig+x0@a^tN&jdB;j1~z(iQXr#F&p1kgy9YiWR4~sL2c|M%9=KC3bk>k( zHP+cwmqvS#!U-*5W*DTCEAaUQu>)PC?BCZRczgN6`P{8D8;H$CzQs5~xq7G)H zgCLo-_kkZ+Fk@B=3dTaV5c}|MwPZs>AC2`8{`GiCvUuW~iAa#41C$M>{V)rjo_l`` zD1_XBwt@@X#tHrphX6d{)z7ASNB3YFl@M+|Uky*QPVx705yrO~6=F$ge~Z!gonMnK z*m+rZu3h`Wt)1mZPQ-^=BamV7^Wj@jU+o2BpKrM+A=DY0wYzLse@`v`KDO2f+J>io z&^cbO139?<=xz=V^~|w2zDTlXNxM8yoPqG|kOYd&FD-tCIHJ|4GpL8YW~ra~lHXG! z$`Rg>q05=yIm0JEy#>zIKRlVy%Mkp=vwmdZIdOj>MDuUrc9V&z1E60suW>?0wj23c z2u2X!Bz`7XOMmgU+-@lP-X@t6$yd3^$ENT6ZU5OTm0-E5fuEgyZ%=XP6>?f$DdgjH z^ri!Ubo@YPr&uZ`vxD{fEc3F(-sR;)vw9`X_9$ibMaaav8|-qni`-h$a#2k!!%jSZ zhsQg}Y_=RCrXESGxaI|n;5;ke&b4s~MzHCd?O9nFdCw@O5{ZvaafSs0c>$kG)gnAk zHogR>a$6|N@cKq190(59leMc_5t6<@@A*k1MR?|to9pvc1MAU>p*Ap|tm-PfSNWZy zz%<1?=c3qdF5Q-~gnH1=9%;FXwO8F^Ec`&$un$Gt0;`R{q-Z3UT1elro+)PDl2aUK zsKtlU5!-?GiLURL8a!E4A@}i-RM~tPg|9|{U~K!t!b?>CsWm#Xibf3mB|J0tS7QxT z*899Sx@CQ_@tEWH0cU`X(k$hJ-*?24(mQdeQ+RshYVssJr*>UCrca%8LjwdQZ}r{* z^fKYz5%Y2H(%t;A@az#9g>MF<%=Sm5k~w)HkXtCT>sXw_j}`klhptE0HBt1iZ(Wra z6#J~yBSRsSt8|prq<^p`Md{uM0 z{~P96na7gRUSA<>rrJ!6k#0bxPepX5nd_siu_7dMTX}`xoJ+rXIPZ3J(KrSiOsGW34Q3b(rtXI zqgQIags4WMkk!V8{XVqWw)|L`nAvec@5a=Sm*O1gfd$pB@aEyzRJX5fmW~{tD!bAw-`_4nSzmJ;hnj3zwkF~9{LfE&4m3~7Qs2ap=9=^a=9j4|mHVy3g zYd4e8+=RW(0!9XZmesEY-E3z7ia3gaXb?=R=kpnzhxNaxPs=8EO26o=8(M}e;u)Cu zzv~M9)yGpJ%xp-pARo|?u6M*ZSyb8@{EjcfN@R9i4h8!x4yj({P8TI zco*+O$VsWw^v~`^Y=%W)^rH{#+L;^;ZhiLP>=(`t=?7{U&<@efA2W>&=8P@vF-v@Z z*>FC{9`?8bkqHzYMA%DbgCy{j!sHQqDifW3E`(2@z*Q@ux&r?B0vtK#@Omt~GyCnl zruJr8>ew!%16PSf&MGvHzKA|_b8%yI6`VV${O*)w zGz)&r1)28rbQTnY{$40wql zu?kk-`+*~kN-I1gnyvg&2nkFEW4vf>XZeJ8@iNl1;#o$3g!B21x5-06r4PH3F-JiK zhCKBR59reedg;#q2bBpO*{roH{rxLB{mu_fByIIVP$TVXLozAFr-$IX_S@pjLT)=y zwy=wj_6zE21{gfXQO)5k0LIM#JWEa2&v>!?i5R}%&NX9|rvbSq#zGlkK*BD;Ord~) z@S%fK&zUf3Hawwwb*P2*f~f!q2UU7<_wETs>iOmOrsU=Z_hNG*=kt#@(v`*LcMnlw zN=^W#y<-2QGU6|4?O^~huQzAKwOr+(2OyBkjzjAL)g?#JeomK^M@l4o0!GyL@Gjs| z6Th*E)Fuz^A7s{I03R}w|K@`RfcIjVW^c~;POCs^mV^e*5E=F!XhW|eNDd7+@^>A0YYOq9{ zouvGdh36R8a2B;Fc67_nj_L+{m=XHnhBj{C^g_S%zj;}tD#OZ5TeOtX`PXZ1%D#-s zmqgKfR5lc}w~{gvt}%{`9pWXQ4MrHbBM9{#x$Nsk zk282yDPXIwt`aTN+MnMCmiveA$oGNGEt*8jkMlDv&v*VP-;3j3#7h7bl z2af8BdCBx!7nbEEY~Lw*qHzxKKEhLBX>S;8SKIkSvP4pKu1a+V#ySB~@fk7Ys-V9}F1LSRekG K2u5+7P5&2!_t=8~ literal 0 HcmV?d00001 diff --git a/engine/testdata/selftest/BackgroundImageExtractionSelfTestCase/htdocs/index.html b/engine/testdata/selftest/BackgroundImageExtractionSelfTestCase/htdocs/index.html new file mode 100644 index 00000000..76f7c904 --- /dev/null +++ b/engine/testdata/selftest/BackgroundImageExtractionSelfTestCase/htdocs/index.html @@ -0,0 +1,11 @@ + + + Background Image Extraction Self Test + + + +

      Background Image Extraction Self Test

      +

      This test finds the background image.

      + + diff --git a/engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/goodone.html b/engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/goodone.html new file mode 100644 index 00000000..ef8dcdb1 --- /dev/null +++ b/engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/goodone.html @@ -0,0 +1,12 @@ + + + + + Good Document #3 + + +

      Good Document #3

      +

      This doc should make it into the ARC.

      + + diff --git a/engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/goodthree.html b/engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/goodthree.html new file mode 100644 index 00000000..ef8dcdb1 --- /dev/null +++ b/engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/goodthree.html @@ -0,0 +1,12 @@ + + + + + Good Document #3 + + +

      Good Document #3

      +

      This doc should make it into the ARC.

      + + diff --git a/engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/goodtwo.html b/engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/goodtwo.html new file mode 100644 index 00000000..bec23389 --- /dev/null +++ b/engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/goodtwo.html @@ -0,0 +1,12 @@ + + + + + Good Document #2 + + +

      Good Document #2

      +

      This doc should make it into the ARC.

      + + diff --git a/engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/index.html b/engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/index.html new file mode 100644 index 00000000..cc6e2db1 --- /dev/null +++ b/engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/index.html @@ -0,0 +1,24 @@ + + + + [ 788219 ] URI Syntax Errors stop page parsing + + + +

      Pages to test problem "[ 788219 ] URI Syntax Errors stop page +parsing" has been resolved

      +

      The issue is about bad URIs in document BASE: That good URIs inside +the doc are not being found. Below are three documents that have bad +bases. Each doc points to a good doc. The test is to see that the good +docs get found in +spite of bad parse of BASE URI.

      +

      +

      + + + diff --git a/engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/one.html b/engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/one.html new file mode 100644 index 00000000..b37c03c7 --- /dev/null +++ b/engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/one.html @@ -0,0 +1,14 @@ + + + + + + Bad BASE 1 + + +

      Bad BASE #1

      +

      Pointer to a document that should be found in spite of +bad BASE: Good #1.

      + + diff --git a/engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/three.html b/engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/three.html new file mode 100644 index 00000000..0a445e91 --- /dev/null +++ b/engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/three.html @@ -0,0 +1,14 @@ + + + + + + Bad BASE 3 + + +

      Bad BASE #3

      +

      Pointer to a document that should be found in spite of +bad BASE: Good #3.

      + + diff --git a/engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/two.html b/engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/two.html new file mode 100644 index 00000000..7ae7b9be --- /dev/null +++ b/engine/testdata/selftest/BadURIsStopPageParsingSelfTest/htdocs/two.html @@ -0,0 +1,16 @@ + + + + + + Bad BASE 2 + + +

      Bad BASE #2

      +

      Pointer to a document that should be found in spite of +bad BASE: Good #2 ONLY, in selftest, +using above base, this document will be outside of selftest +scope. Thats OK.

      + + diff --git a/engine/testdata/selftest/CharsetSelfTest/htdocs/index.html b/engine/testdata/selftest/CharsetSelfTest/htdocs/index.html new file mode 100644 index 00000000..60536937 --- /dev/null +++ b/engine/testdata/selftest/CharsetSelfTest/htdocs/index.html @@ -0,0 +1,10 @@ +# LocaleCache_jp.utf8 -- test data for LocaleCache in Japanese +# (Translation provided by google; it's probably not accurate.) + + +ã“ã‚“ã«ã¡ã¯ã€ä¸–ç•Œ! + + +multiline: +ã»ã¨ã‚“ã©ã®ç¾ä»£ã‚«ãƒ¬ãƒ³ãƒ€ãƒ¼ã¯ç§é”ã«ãれ毎日æ€ã„出ã•ã›ã‚‹ã“ã¨ã«ã‚ˆã£ã¦ç§é”ã®ç”Ÿå‘½ã®ç”˜ã„簡易性を傷ã¤ã‘るパス㌠+完全ã«é€€å±ˆãªã§ã事ã®è¨˜å¿µæ—¥ã§ã‚ã‚‹ã“ã¨ã€‚ \ No newline at end of file diff --git a/engine/testdata/selftest/CharsetSelfTest/htdocs/link.html b/engine/testdata/selftest/CharsetSelfTest/htdocs/link.html new file mode 100644 index 00000000..e69de29b diff --git a/engine/testdata/selftest/CheckpointSelfTest/profile/config.txt b/engine/testdata/selftest/CheckpointSelfTest/profile/config.txt new file mode 100644 index 00000000..e69de29b diff --git a/engine/testdata/selftest/CheckpointSelfTest/profile/seeds.txt b/engine/testdata/selftest/CheckpointSelfTest/profile/seeds.txt new file mode 100644 index 00000000..68317425 --- /dev/null +++ b/engine/testdata/selftest/CheckpointSelfTest/profile/seeds.txt @@ -0,0 +1,10 @@ +http://127.0.0.1:7000/random/ +http://127.0.0.1:7001/random/ +http://127.0.0.1:7002/random/ +http://127.0.0.1:7003/random/ +http://127.0.0.1:7004/random/ +http://127.0.0.1:7005/random/ +http://127.0.0.1:7006/random/ +http://127.0.0.1:7007/random/ +http://127.0.0.1:7008/random/ +http://127.0.0.1:7009/random/ diff --git a/engine/testdata/selftest/CheckpointSelfTest/profile/sheets/default.single b/engine/testdata/selftest/CheckpointSelfTest/profile/sheets/default.single new file mode 100644 index 00000000..e814173a --- /dev/null +++ b/engine/testdata/selftest/CheckpointSelfTest/profile/sheets/default.single @@ -0,0 +1,163 @@ +root=object:java.util.HashMap + +root.loggerModule=object:org.archive.crawler.framework.CrawlerLoggerModule +root.loggerModule.parent=reference:manager +root.loggerModule.dir=string:logs + +root.seeds=object:org.archive.crawler.scope.SeedModule +root.seeds.directory=reference:manager + +root.scope=object:org.archive.processors.deciderules.DecideRuleSequence +root.scope.rules=object:java.util.ArrayList +root.scope.rules.0=object:org.archive.processors.deciderules.RejectDecideRule +root.scope.rules.1=object:org.archive.crawler.deciderules.SurtPrefixedDecideRule +root.scope.rules.1.seeds=reference:root.seeds +root.scope.rules.1.surts-source-file=string: +root.scope.rules.1.seeds-as-surt-prefixes=boolean:true +root.scope.rules.1.surts-dump-file=string: +root.scope.rules.2=object:org.archive.processors.deciderules.TooManyHopsDecideRule +root.scope.rules.2.max-hops=int:20 +root.scope.rules.3=object:org.archive.processors.deciderules.TransclusionDecideRule +root.scope.rules.3.max-trans-hops=int:3 +root.scope.rules.4=object:org.archive.processors.deciderules.TooManyPathSegmentsDecideRule +root.scope.rules.4.max-path-depth=int:20 +root.scope.rules.5=object:org.archive.processors.deciderules.PrerequisiteAcceptDecideRule + +root.uriUniqFilter=object:org.archive.crawler.util.BdbUriUniqFilter +root.uriUniqFilter.bdb=reference:manager.bdb + +root.uri-canonicalization-rules=object:java.util.ArrayList +root.uri-canonicalization-rules.0=object:org.archive.crawler.url.canonicalize.LowercaseRule +root.uri-canonicalization-rules.1=object:org.archive.crawler.url.canonicalize.StripUserinfoRule +root.uri-canonicalization-rules.2=object:org.archive.crawler.url.canonicalize.StripWWWNRule +root.uri-canonicalization-rules.3=object:org.archive.crawler.url.canonicalize.StripSessionIDs +root.uri-canonicalization-rules.4=object:org.archive.crawler.url.canonicalize.StripSessionCFIDs +root.uri-canonicalization-rules.5=object:org.archive.crawler.url.canonicalize.FixupQueryStr + +root.order=object:org.archive.crawler.datamodel.CrawlOrder +root.order.disk-path=string:. +root.order.checkpoints-path=string:checkpoints +root.order.state-path=string:state +root.order.scratch-path=string:scratch +root.order.max-bytes-download=long:0 +root.order.max-document-download=long:0 +root.order.max-time-sec=long:0 +root.order.max-toe-threads=int:1 +root.order.recorder-out-buffer-bytes=int:4096 +root.order.recorder-in-buffer-bytes=int:65536 +root.order.http-headers=object:java.util.HashMap +root.order.http-headers.user-agent=string:Mozilla/5.0 (compatible; heritrix/11 +http://www.archive.org) +root.order.http-headers.from=string:info@archive.org +root.order.robots-honoring-policy=object:org.archive.processors.util.RobotsHonoringPolicy +root.order.robots-honoring-policy.masquerade=boolean:false +root.order.robots-honoring-policy.custom-robots=string: +root.order.robots-honoring-policy.user-agents=object:java.util.ArrayList +root.order.recover-path=string: +root.order.recover-retain-failures=boolean:false + +root.queue-assignment-policy=object:org.archive.crawler.frontier.HostnameQueueAssignmentPolicy + +root.controller=object:org.archive.crawler.framework.CrawlController +root.controller.sheet-manager=reference:manager +root.controller.logger-module=reference:root.loggerModule +root.controller.order=reference:root.order + +root.controller.credential-store=object:org.archive.processors.credential.CredentialStore + +root.controller.frontier=object:org.archive.crawler.frontier.BdbFrontier +root.controller.frontier.seeds=reference:root.seeds +root.controller.frontier.bdb=reference:bdb +root.controller.frontier.logger-module=reference:root.loggerModule +root.controller.frontier.controller=reference:root.controller +root.controller.frontier.queue-assignment-policy=reference:root.queue-assignment-policy +root.controller.frontier.uri-uniq-filter=reference:root.uriUniqFilter +root.controller.frontier.rules=reference:root.uri-canonicalization-rules +root.controller.frontier.manager=reference:manager +root.controller.frontier.delay-factor=float:0.0 +root.controller.frontier.max-delay-ms=int:0 +root.controller.frontier.min-delay-ms=int:0 +root.controller.frontier.max-retries=int:10 +root.controller.frontier.retry-delay-seconds=long:90 +root.controller.frontier.preference-embed-hops=int:1 +root.controller.frontier.total-bandwidth-usage-kb-sec=int:0 +root.controller.frontier.max-per-host-bandwidth-usage-kb-sec=int:0 +root.controller.frontier.force-queue-assignment=string: +root.controller.frontier.pause-at-finish=boolean:false +root.controller.frontier.hold-queues=boolean:true +root.controller.frontier.balance-replenish-amount=int:3000 +root.controller.frontier.queue-total-budget=long:-1 +root.controller.frontier.cost-policy=object:org.archive.crawler.frontier.ZeroCostAssignmentPolicy + +root.controller.server-cache=object:org.archive.crawler.framework.CrawlerServerCache +root.controller.server-cache.bdb=reference:bdb + +root.controller.processors=object:java.util.LinkedHashMap + +root.controller.processors.Preselector=object:org.archive.crawler.prefetch.Preselector +root.controller.processors.Preselector.scope=reference:root.scope +root.controller.processors.Preselector.logger-module=reference:root.loggerModule +root.controller.processors.Preselector.recheck-scope=boolean:true +root.controller.processors.Preselector.block-all=boolean:false +root.controller.processors.Preselector.block-by-regexp=string: + +root.controller.processors.Preprocessor=object:org.archive.crawler.prefetch.PreconditionEnforcer +root.controller.processors.Preprocessor.controller=reference:root.controller +root.controller.processors.Preprocessor.ip-validity-duration-seconds=int:21600 +root.controller.processors.Preprocessor.robots-validity-duration-seconds=int:86400 + +root.controller.processors.DNS=object:org.archive.processors.fetcher.FetchDNS +root.controller.processors.DNS.server-cache=reference:root.controller.server-cache +root.controller.processors.DNS.accept-non-dns-resolves=boolean:true + +root.controller.processors.HTTP=object:org.archive.processors.fetcher.FetchHTTP +root.controller.processors.HTTP.server-cache=reference:root.controller.server-cache +root.controller.processors.HTTP.credential-store=reference:root.controller.credential-store +root.controller.processors.HTTP.midfetch-rules=object:org.archive.processors.deciderules.DecideRuleSequence +root.controller.processors.HTTP.timeout-seconds=int:1200 +root.controller.processors.HTTP.sotimeout-ms=int:20000 +root.controller.processors.HTTP.max-length-bytes=long:0 +root.controller.processors.HTTP.trust-level=string:open +root.controller.processors.HTTP.accept-headers=object:java.util.ArrayList +root.controller.processors.HTTP.http-proxy-host=string: +root.controller.processors.HTTP.http-proxy-port=int:0 +root.controller.processors.HTTP.default-encoding=string:ISO-8859-1 +root.controller.processors.HTTP.send-referer=boolean:true +root.controller.processors.HTTP.send-range=boolean:false +root.controller.processors.HTTP.send-connection-close=boolean:true + +root.controller.processors.ExtractorHTTP=object:org.archive.processors.extractor.ExtractorHTTP +root.controller.processors.ExtractorHTML=object:org.archive.processors.extractor.ExtractorHTML +root.controller.processors.ExtractorCSS=object:org.archive.processors.extractor.ExtractorCSS +root.controller.processors.ExtractorJS=object:org.archive.processors.extractor.ExtractorJS +root.controller.processors.ExtractorSWF=object:org.archive.processors.extractor.ExtractorSWF + +root.controller.processors.Archiver=object:org.archive.crawler.writer.ARCWriterProcessor +root.controller.processors.Archiver.server-cache=reference:root.controller.server-cache +root.controller.processors.Archiver.directory=reference:manager +root.controller.processors.Archiver.compress=boolean:true +root.controller.processors.Archiver.prefix=string:IAH +root.controller.processors.Archiver.suffix=string:archive.org +root.controller.processors.Archiver.max-size-bytes=long:100000000 +root.controller.processors.Archiver.path=object:java.util.ArrayList +root.controller.processors.Archiver.path.0=string:arcs +root.controller.processors.Archiver.pool-max-active=int:5 +root.controller.processors.Archiver.pool-max-wait=int:300000 +root.controller.processors.Archiver.total-bytes-to-write=long:0 + +root.controller.processors.Updater=object:org.archive.crawler.postprocessor.CrawlStateUpdater +root.controller.processors.Updater.controller=reference:root.controller + +root.controller.processors.LinksScoper=object:org.archive.crawler.postprocessor.LinksScoper +root.controller.processors.LinksScoper.scope=reference:root.scope +root.controller.processors.LinksScoper.logger-module=reference:root.loggerModule +root.controller.processors.LinksScoper.seed-redirects-new-seeds=boolean:true +root.controller.processors.LinksScoper.override-logger=boolean:false + +root.controller.processors.Scheduler=object:org.archive.crawler.postprocessor.FrontierScheduler +root.controller.processors.Scheduler.frontier=reference:root.controller.frontier + +root.controller.loggers=object:java.util.ArrayList +root.controller.loggers.0=object:org.archive.crawler.framework.StatisticsTracker +root.controller.loggers.0.seeds=reference:root.seeds +root.controller.loggers.0.bdb=reference:manager.bdb +root.controller.loggers.0.controller=reference:root.controller diff --git a/engine/testdata/selftest/FlashParseSelfTest/htdocs/index.html b/engine/testdata/selftest/FlashParseSelfTest/htdocs/index.html new file mode 100644 index 00000000..4faa8da3 --- /dev/null +++ b/engine/testdata/selftest/FlashParseSelfTest/htdocs/index.html @@ -0,0 +1,32 @@ + + + +Flash Parse Self Test + + +

      Flash Parse Self Test

      +

      This page causes download of the swflash.cab. Is this going to be +true for all flash reference?

      + + + + + + + + + + + + + diff --git a/engine/testdata/selftest/FlashParseSelfTest/htdocs/pirates.swf b/engine/testdata/selftest/FlashParseSelfTest/htdocs/pirates.swf new file mode 100644 index 0000000000000000000000000000000000000000..209800ad274269d78688aa751f126e8ace11de12 GIT binary patch literal 18954 zcmV(nK=QvsS5pUWO8@|PoSeFKP#n?MEjk4EU6|DWTf3qXK@ga=I-I?6&w;87XBk5G9mF-QgTXaT6#`yUVcGgQE^F4ZC!msV^ecWPj6rU0CaF@ zcxrlPc5Z%QacN@{zO}uB*xlPdJHNQRy1u!+yZ;X_Bmgqf|BC;T7r{SXul_9y3i^L| zA-!_{&j11xR4R5fLNOKeZ;nLN9Dx|b;_=zl-I!ob)l-u1PLo(~Xg+Pwp8bc~|77;x zBNp`ki`oAs_J8wQ0^lGc{aZX_0)PnMfi;r;&#V6eII+Onp9-g0VfH=}rM2Zr>7{bY zuU4v*0W>BFUB+D7nv``Xy)tnexK`ht&+|*ngOA|{FM#>YMiHyAwbMUs3RQ8Aw$sx` zDrTJ`J4XCLe0xU+&;FIbq1N3o@QAI-3N7@A?{w}1m6}c}iY6{a&9@q)uF>Flh}emy z57)!zN5Oe6nafN5{KUa-3i&1iH9n{^g5trgGU(=rtQ#y+dG{oeaXp<#J%}-!fuIn9 zq2&*2tkmWD#?A~gMc;x;R;>N8e1wYizkEoS*N<99%8VN1y=QS#&olg_#%?vj*g#4sfG;uKi&QUj=B}0oTEs2{x zzg^xh71;CK+Uq(#LDceFPQ_&@U69fdmpSjEAoXtr@ z#L)1xqEWe}xijK?K))+?0l1n1dcQ7yq)%`d9fd zl^#H3en0RJSa&iv@Z4D#+IgFkR>Gpq3Y6Y>BqP4Ig$Z~sKNQo&Q>)do9zL2R0ANnu z&!n1O6shtI*7FkJk$7@I8Jp;?Gh{a3)W&q7oyVu0|Jm>dOv#{`?g#bx2ZA<#Zka8k zSe5(O`FyKeF&%B6(YGoi?0V$p$aK{QkFA=&3KZFPgi|m+bU zCGCv+%1tgdI7XM@n+jjAv7iH+$o(2@0;}Akq$ZF$B}{#h7rgDdD|R$U<0ri0nljpC zJ4X86$t!;P?DB~z_3Ts6)CkOC<#Bv$@^13J=!q}7E*7OJ3-~-}1XnI-gmUGkFRsXMk){@NC z>zZNz&s?Vk(Qem%Ku;Dly0GVAx+{%Tn>@R<0#HeJ(`Oj|e{bQJZq_0-kjZW!Ai17*JckyjqZu$9L zc|W||-U}0MOPp|ny^lJZEyRQ-)z0>QXPPb^O`6YLxQQ>)()aME zinmiH3G?Oe?TmDWF0v-hQc-thb)N#rsa!uWeV=wX(P( zXi#b3fV1xIduO%V87s!H?$4&gNp8{y6L9pV`*R^L&6r);uh+I&N;sb>D6LW+`HIUt zMm|TX5OV-ii8^t6uAsb7L1!q%{j9+6-$K^nO%K8{CiGnlIArL^qzBV4fXcZlACX*` z3ib=YNJ|kO58fx`Merhv6c*0aAKHT6FfUouyCLZd!q~_OV!VQ(;zAetio_eH6i=FW z8LvwU%AayH{FE-t>-@Sk-m>q4=x1ptcW=~|C4omRTya&jzynS_x}H>7BhymmT6fei=U|cf)dGq z^-Hvp`FH`|ulxdL@6Xp_y?({5c$mv7W3B4J1dJ%wT4U|?PTARo1n^SJ97!nnRnu1~ zJg^A*zEjERa(kgwmiAk2c#pX)+m0%0%esQ_2~ov+vwSo(dryc)Hsu-qZmv?N6Xw-< zF6=LM1L-DigkrtU{Ucg>J9{f}Zsmr*3>A-?H&b;SwK*klUimg%E%dZwuh%y#5MFQO zuqQXY4)7e5Bu5BFRa1tyJ4%r7C++3eYFcl3B{|)@u&JBVR&wP$QFKx$UPsbbemTWj zs$B4|WeO{(;);j}?HSL`wHWM0Ll43je1BCr&Lod%dvx{tgSPv!Ax48-J_@P?@-M+K zIErI%oR{XVeKew~?zAA+zj*jtea+7!`}aWl@6?j--Kl;VxzgqPxfv+1hHeU^DqLzp z>{Fa+CS-R2$AODHptAhoYN7c(y{qF(X}33%da%x2t!Z1q^|uI3rf%F3KMIj#f5yq? z>iU{$ZM$QFJL*qZQpam4ioU$?Am_3xjCD(jE{fjGOy~ zBKn^f0N+XEMFt(i9i?K)bf`eN_S7RgBbQ|c+E@CO_#lsywZ_X!n5nm8p`_E)aG8ma zit&n~^Eeaik!Z8Y83P%FtP{D|B?M`lnbGmn_za zf41mz9L$#;K*I-u4<0*(oweqT>f0au(!7aBx-ylG9S_B?VC47krOZuA6834*itW_s zV(6-H?plIB((S0jg)P@iOKJ;yNX1ntRdgfeyBo{)X?zr;fy*16Z6)O9#K2ZhZzlc+m_D*bV` zPYkVheeh_-xJEjab0V4YcK)4-fd3MA%W$_MJ7KAMdBR5Ci zX?cP=q|upeY$r19{yy|IYR)@@rHxQCYmDN4G?Ph*EHNtzK=`rl3cIfH3vT5V>V9zy zQsM2ds%NAy-M#^zR-#P#h&UuOpd_<4%uhe>I|l<%V=#^U)OsCc1YIii&ZPTZ0C*PZ z)?dn5R^bfi%oDcm-{LQbZqTIFV0y??ZKEyuyozc?NBKz)pugT&N!g0gsJ^;xCr^6E z^pzr~_WcS(hLEMDP0UYs$EN4G%p02*yi1`~Sha-W+oCkK=+5-Qfdz$i}#5H zqvl~RfI8vIvfHd)H#*I4Ts!f$gmT?AC8EnzR=zIEKcTKfRH1g#$r=IGnT3cT9< zSdRK+{)Jl}Chls#%|FxFm~OcW`NN9Kaha47Ren8XZ^g&3MLNz?yZ}G}=BGPVNsD9^ zvZNx>E?%EY*(9=3AsFO|mnZWVT=}1;CNhX~Rddw2rE@&wdwEoP%<99X8*Fko^oCO2 zUkY^(4qzfE^X(KiZj#crram>#prLKYF|q?63XQ(wN%K-j8$9~`-aJy02>eV%AK z!cS%ohKX~|FFY}k$u7AHINhGlNt)dZVBDL>`1g_=Dn})8jWwU8-dB4>Fna4fVce3H zdYC~sV!{iS40V?8R7fneuj3B;7uMlHS{au+?Vt|DTgQ^8T$wPfS)Zbr*HJP{0&wye zC4-d*MDfIgqVuSWbK--mihGwV>gsK|l2-FcV$IB5 zL!#^^jF#loLDE;W^}cXWKAAPr#Jb%+Q`z*RQnsO@QR57;94|JrErGqBJ+eqB-J7t& zw=)YlkqpHDWDnr~gz6dU?Nl9w#8*aA?WG|;RnhT&mKd4KtIABONgpA1;nGb7PSp#8 z(;-i1l+l{(WKol4GKa3}hZ6PJ&i9&Awf)4WZA1eJ{nWo)Tv4@Ux=}!X=7(EQ+sh04 zI^QgHTN5ioOII**KVy~TXdr#X3vla#pOS2w6+Z~9DMBYq-p#tL>|PH3sn;rOG?h>C zXEzz&%nKe`%?RS0=YuIWAu2C3#qXcVSl%rmzIJxtl6ykf+{#Q-P#XfiGG}qJkE3}~ ziYPe*7?8OO^aed3L8m+YcoCg6-%C8ylx^_`=F%Io?P=!bqPaEDQM>7f9<|Vu1x;K} z`hA`$3O^NGb`;Lfs$M%4D;T&<-yb?IA%Y$)H-IifTtm$L!C1)z?)vb4`P1`u_Ot3kQ z2y#MsY3N~sMlpEDys#f$%9p~aA1b%Jn6hJyne({vl>AIS+sNMwf2F?w`~`LT`80iC!(J&FyBrEk8OwE~sw%CBPP8CQ6)`5m5F6 z0Y;9o(w8d8+Vrf}8@f~~j6d%v&mXg-foRZ13YX=<$$kF9cm1x@6o2svxFNJn68+6@mkrN33+En&I+(3ZCb=Gs+Ym<3ISbz%)f&H z26_v`5bIR&n!ElN!0hq(7mqF!UxdxmgqDk=$Mq%~I{mi9ue%JcQHAv?mND!(^h-EKMFVi*6xC4)d?lM1;jhJKr{n^bZHDRknQf^W>@ zrbnD=_NtX@A+0{flf)FK1R)RrMLsbGKnfS|RVGEl`dL5fr!%mfF690B^b<5dht zP6jQ)8pZwgt1qsMM^tk2rd(zgQruHS%@#g~%kG?Gvd^7uuPKFdxlg3op}B;fIlrJSIiZ4_tSO zDltM{1b3Pxd+ygQnIW2Usu4y>5={78zPLWc@Mv*EtXsQr3VY!iKP@k_`QF|cdOFSHv1vE7-%H!-)7F?`F?XLFzL{3nu1Z8QLyKK(w6Id*ZQ_KL z)q8$^0d$_;mY&>lxOVl-Qd{}h?V-AjNhWLtKgM}naeMyw<=HjDr9gH|bOoJ{*7xrI z={Rd^e=m`&oGF7m5Tf`Mlqw{x0yB^{n4ry$(I&1Ics7BlK*-Ku7e~RdDG4(V?})zs zR?3bqwa}=PAW?$|=mcuGDafn*AN zFpBrAzLc2H_v+|D?bKpu0iWV5_ycNBkn-oq(%K6pj($AD4CH zs0?R)EtkL0%(PicdLHZj7rO6vuIn$M9W!bB743QsudWPX|uuI7f z=t7`)xfJcI!2zu~_5yZu-`PehCW6qMIHXmSZ|D1x=lm#Q#s_nA-!_`5^}dO|al69e zoZf-ivH4N1>GFghZT^xOSTe6_iQ$F59?zrs9dYMWPVJz?-sjP8vsX`6J1RHCVMuIw zt@&1Uu$ZLrIuhBHfQg-jy6Ix8yC=+TNpz5@i%}GL+~oA=D*rBKMagYh5ad#ACN2$f0xR)<$Kuf z1x+$H`eHzbujIAw_jVW_7P@Tnwv4WHc5_wASxt4`&bWtM>0;CGylV(9FdR<+RPtoJ zV*{`rhDok;^CS*C$H^S|t!YdTv>KNz`KJ4dkA2aY=7P+|CH*x}7==3^Ir>($}<)axkW&J-@X6q?r?d(5Xq)V=pJ1lRR(pd_Kk^L}k1Y?O`BmEG< zVb_P5n@oJ3cOdXMKWSW195kT66Rc(-&*Py-uIk>x@pi<`8S!Z7nOed$;hw~ZmuEKK zj_=9LK6Q@oz8gFkxXNKmW+cki3CYN}Kwd<1?LM>?=Zlwuo1g_r3cB~&D zO!P+yyd$AQs!-(k3}aB2S@!8#zrghb4*zfF8e{y#I>;aA%PvF#zs5lDh78TW7oORH zwl&irONdi_sZaIE;c~YVC#7h>cDMdY`o&LQTvbZufPY%a?FN45!aIl{$*Z1FwPW2~ z#!gZ%rX4MRqHgRXYh5GKUR1~zoP0_HqW|4L9$mTW8bV(1dbjD4Kle#U!5-x!LG-{T zfGMVs_xiC$xPE_5nTETh2(TnAvG4Vq+I4wPps_!yK;?=BlqSbO{9vJD=o2dD#gA=wp$-(Sd z%8`9}L-E<0{dte`y4$HaL2ukKsB?3wpE{8j*@VFgk1_0-;?~ce=r-PVJ8Q5Zq$ojB zT)x?ekd|UGVpmBNTJOex6r$-B-H>lvq^)hBCuh4EJDRj~V#RnHcoa6*v-sn2M220;cKcqdZ<-LT;|DD-=))se8QjJ+d4 zfc>19fy#~;!H(MEh{`y|d6jWnU&5!*@#Chdbg^5wU_LCWTIX{kyR4127^a{pX{;W> z;Js^6n@aieA{e*@sI*=E=a^FI3b!Mt?D6(BR3xq`)L9SpWE}->&gI?dQiA;5t9ICg zZ8Dqn&UFEJoc%nu8*8{L2adZf-m6EEbP4KdV{NUAQqw$Y(S1qEz~=Wh%08)+h@|*S zN`uIo@)BOy>)TkhrkT8uvW9JJX&Qxd-tKLgt1HSdPu@!V;+3u(`a(10ZE{;>G=8r$ z*q;-Kq|x&D3a4P=@VNpvB0hqqSuo=U!8dd@L|wPJobV71@LA}ASMXlZ3A#Y7uJn1Q z^@_f0K>5Pc$i z%%UI1uGrM9v@e$=EOY9bqz)pdSQUZ!dE?tL;u^>r>|GyA8<3m|-#cCjH6!MMBN~s< zbPBwmya^);8a(o5aU55*e|kdb&%42)9peWXc6ii$C~<4}=TS%M2rJ#gX5PD-s8h8wYVeuAFJO?>jC3zJKl<-5Ll) z?W+=sUUAgT;S_I_@pq~K%5f9#NNxuVOc}oqP%pX49H=!d-_t4hSvJ60UJjx$kI1AB z3C#wY9*DzJT@IaP zn!`w%ra=Pfqv%}zMi=roPoV7^XtB#n{0 zTh09KZG<#|)JO{FC5txaa&@*MUe#VC&7#O{eY517QAw9Re6dp9z#c~XEGbKt{1rQj z^(|Nsk|7-=C{QF{1TDfaeJ#E}`scMSdgBYAa;jyyT!X?{ElYRaH2m#TGD?a26Fm@UqC?*JBK+ zmow=awpLy>AG+yySqY=f9DmCMeyp2K`?lcIPau*Sf(Xpu0j*yn25E)mnrKY%^>7fK z=#_RI!3*=%L#4tg8$aF&aj|}?`boTNhg`?FJ3J{s|FBi9EVJCjl#ys4;{5R7xlI-z z^-N3Fp{j__dxhZTgYuOXiS_diy_+m%fI;r%?_U5^2k}&M3jhHeEM|dg zP*Mu1EerZlv|PRbh}3oVc@MFx`Mx8OL)cF)2Yg#@O1*T`z)=B)PFqOO`IvkXm-p%gAd+%Xak2jPCtl#D$f3ZZmgUm#kRRknVKy87Xmpl0Oc~C4 zGma;Wq{vw*Oe<&8vCo|dZJp=_vJl$90a1#x=`~Ws74$=q#xGizZ&5u^0@)tv#ZQ_- zr^`CO`#@+@_11Adci)(p{Cy`QIq}UHW$x!C3>7e)CFPs%4%av6-nL-T=M=@|MA8E) z13UOQVP|xGRAT1u>W2f;&?A0R#$Hu+SZ@floAvldHu`<&_|a{1A2M%!esca-`a2{v zC;<~H=yJ$W%Br=p+gr$>{m%bedv(hcOL>lr$#e*NuQ!5$z89r5I0?|oNyWF^6g(hv znTanyb*02!;lAAZ3V%^Fal{w8BgqmSP;I0>{+AQRHE!9!VIU8T{Kk?ox2>W6dxvTW zYh3n&Xk}~sg##~(Zi`T&PXEV5ivG4!Hp7}<7;W4$D*%A7)C%N}xsZa-wpEI7a;AVE zTJ*3128j1$=^Q}XDo`;|9w5)g6u{BRZucrGLBXSNVC^lgoHZ`i_zqv$(Toz!{;S_< zzk#fonxq@Lkn$KQq2GNx7dl!BLWg^TZqTt&#~Ahm$F?|9)+s#nzW|f^fA)o`wKbMd z-%y`Ayp`If1cgEfz3Rz%3arMGOk=V0L%?I!zIDtk_|26nZ;!n-_rL7v~{Fc=6t@NEc7an`hQ~8RNHZ)5Ek0 zOA0|B5CKk8mq4bE8+HJ&_J2lfxP16CczW& zgS#eYxUIL|n$^!^mbT>^(b2UEWS+_)7S;5)jCGtEIOlGfyZ<7Hl&imPk3$6e^3k(lmfK%RDSg z+EsSE@_W}jM<+EI^SZgS-u~lz$ofc&y7!OCzrsY8(Hwv+27m(_A6=HtOd2VLjvbCXB%WY@TGlXN)f@xX4nd`Fi!laJ!QU!$cT`mV@b;B|bGN z^l`*tyx(RT5oQ?^^tWz(FxI7+XE6KLs9eAs@@;5Clu>=gryPco06?}8;mnLE=EAth z6%nnf_M}OW52lSS{f2F9&5iV3$opbK{YgaGdj!!sb*QsRpCJjN$o;F0m2r3%aF^0V zgV(lYJYr#M)-X}Klfx+YpeYamh=L0jip*yxs-NLy<-Dn@jv623{FpM(3wd%5h9tG; zAw2f){nKXYP^G48^nFgICqZZM0}6MEst7KDAC)dQaZ3{P@envNOZ_?0l3dVRX0*wP zbDvGl=TOJ$#?rD7_X9|0<-vlcTCPW-%9(y0+O^hB5PPZ-&Uo>`2&JNMX2gf=nM<*! zOitYla?bUwS9}L2!P;u&q(cQRlf6UYAC*$B7B)tszP>^TVcmpi}s0k3nr?<;0> z-Q%jGf6xhacRfpYib30LB}3dQoa7RtVgh3PQ|_)tPqh-<-H{{o8psYHE7vaoL5+-v zY1@qBT=fKfCHWwytz^CxHYLiGX;SzLU{+24Pcl_pK2#6yuH*_qd;PBemqRk)El}h70DG64=!Ju|ac+T}t z+_9R`v3Fa^jMzv4%Z>qFj$K{gn50eNH$fsbiOH)8WXPnHS$A-i=9q?5Oh%>zI*>Hgr<3G(rc9O?w}?h?Cx3g8s6b@ml?~1AE6vN-ZOs7 z+#Ol^N_rSwB})o|r$kw(cbTp~Jh%@SaPYHpa1weNK(D$)jD2c0_e^niTFG3eSb4Y} zI=KTS&1qS&gX!^PjJsI|;gtp%S4Z72xy^FP{#~yo8}X8gOMV5pVG$;O)G!RdMkgyD zJj#jv1>kc}7cBS!xRpGDjC=_ju)I98OEEC{(mRY-^o|=)(+bQ3l~i;JA8P(uv8o=y zrcx+x3_&rb@n=Oh6kGV-E^_TY^E_`^Wa-4x75`zW;t)uqK{iQDD7w|$HHdTATVb7$ zrxVUz@In5wwZWc~e8=COYk_cXKlMjPiFlhnlK5OkD4!}ImO@H5KG0fv3K6W({JBG4 zm@1Y%eS;mV$mkZ>t?GtK$49!WNm9A3WLnhAnrHKdcLh^$=k6!UoxWBDCNdbEEV7uH zuf%{g?C)^zNQ`v)^Y`z81jHjdxJJal>rKz3Iuj}&k{tmJf zSmp(h&C<)&C6LplWe^U0QilO@;C^#lTmBR_v!JN~OpR?S?E)#824u1-?9dgd6)NBm zvqbfGtob?#ayR}j-2Ac$7VBfROG$SzlQ+Z=9>V=my~B}=_%<$2^8sAdse}Fm9c@|8 zW$oT;j|z_n4|M=w6BsK;@@4!tW;S&(vQ=#x5g6~+;d3CgsEsj`$ zPG{_X+2LXj`Q!NDN(lps+{M-1`(I0D*+@Z~m_U%A1IDqC6niiXD0I8T^=nU?MP+;A zcd`0dCP~>h2Xp`qzM)c2nV>%_J|N0|I~Sb4Y2<>Y@a6Z4oR431~uF>LQ4388nuH=_nlI2PwQ?0Ev;8N{d+oLA^RDJzh~~ z-PT@$%bawSd*rwQaG)uAkXIp)1y-IiYb{+XY%JJYEU91@n@gV)d5xcpb(eVkZCOC74^Ii<5yv zF3*Gt9ffrV0bGs>lJrsWWs6xDWD|QtM}O}p=${xlo>%6gTsS_gM#;TfOE_8ZD$2nF z9t#tEI9sv^yCXq1357B%i5=a>K%T5fIkE#yHR)JdW1){v0`JmEDi+LcN^#1zls^4Z+c!Y-+onXBZfYTO+nK8<5{48+m$E3d3%EPi!1zN8y-pgP`zWv`-s6zlhvZ{W?7h zakOs>U4()8wjjRJ7ogy!aW06J zV||RXUI3a3Ox8+K)hleh74y4J=xWoA_FbAnvNT@57=&242Ugp*HE?v@ZOmgFSG+Iw z`4b94PnVQxZ=;sT(v03^eCGucjjFh3e1>k*yozPaG6=7C;wF&NR%Uv`%P5dF>iQ_~ z&nGWnW+Kt$uaxXHjK!Ctd$9$*8}s z$Q%;@MfRTr@T$mwa1&GiWV+D*2yw@hW?cc z%d(vqagLGbu1e&2AuqRsXW)2{fQ4%+TFJSz1?GhTdy0DZL+*T`0cTW{iKcF;&R}Tv z$g4j9_3FfNMNe3TZ@ZIq#*E93g>!q}(0-r!a%W@ZcI4&W*Bu>*8xO{2``_?V56zy& zhR_fE<-gLct1Rq3m-Y22BE5GdkIFzgh>`R*2rbasZE^W&`99f`MDU<Q z?*@^6_jAMW=RmkGUSheOFI652dBWVW7|`HB38=D*fjIHSF0`8R3Ny7hn)>TbX_m^j zg0myb3+H=$ zs*Q6Vy?p_6?QJso^RE$DvHjtmCP$j>kfL~HDmJ()3kEybM!(yTiUB7$Qp>(xk|Gs0 z$=W7U#IL6C*ovtcE7!41H<|vcmnvC=y3JTPI>3hR+zK!U(l)^XcXcIB_1~Zn4m+}y zySeeZoOF3v_&F=p_h@Wmrz#o}Ub;?lKZ&Td^fLlcX3lBIBJ5V(o?!VyeevJ@qGj{u z%pDtUxs&e;F!-Hcn?|l}(b8D;@i8|$R$=^I;LyEytkq=_J9%c&8{bJ%m1Z@tF5}p#FUhN-#M!ic~5(y+i5b)(`-pfjL`XLz}v5& zoVIMkNuD6;5pfEh!Jn=yE{CHcr#mOUdcbZ7$@U1>Z;aTBu8l^r3;21wh_`vFY`gD786MwrO1$ zLcT(03;tAwl734%G(TSe7&j4{QNPq^qeis=q(Ns$Yc!rKm+N?skK-ZeW`1H6;r4od zz()a>;xg-y+cG^0X0!=*2w>MQvz5{gl3LCpn4QR{79Ssf5c2}~;{3$K&A^fEh1*PS zjxOjp&qryv=e64K#bdeY&HHoNV_(?{{@9|lh+(?(qPo*Y2vf)gzx{wPoFa|BOR_X zbH(>GJ=T5V>B^IGk`0Bj$$@^AGE z^hs3;!!;$fxub`paawMN5MSPfck4a`Jt$_mq7GNAgwsBjQGW5vLoAzrl?_uJBMmI3ih2H#43q( zL8t327i7NTWboeobcSBRV|z(P)Ln(-h~}V@-`QP*%B@VfPE* zwWZhRmhr^0I)`X+6mQx5RH^C_AZyxp47~)b#4e`bn`p;z!AXm~inR|vuJ{VxnWfWt zr{zU2u@rcR+35%1Ca-v}gMux=QSh`F6n^sQU9SYe>49iIwotQFLR0<4m61m|3~+Ep z#`ng)cFixfr$+6;s#E2YdD>I`Cfa5#Hb_+{yx^$hT|Z@fy;B$*pAQ}@*-&Tg&!}#B zwEdMOg7T)Es4+E!8iHw*BfgRZ+)QFj{Fz|r8D%BTVtwmx7}&X4&VOk!r@F>G;?5tY z!U;I1M+(Uca(Z>j9)#x98?UD(Ph9e>f-2!a;&#g67?W9h_kdNs*{-Qk)6^KbmpcPG zhq^Bt_zng>s}rop^#e@7Zj(*!1`c(58ocpoS;!gLWR6cF_q`AG<@~l%2wTjCF^vdw zSwgMRA5wTqLcWd=JRvWa%bj{(mhx;4;ODqAs1&r4+!3l|<2dFLKSpl6v$QX>dVY?^ zJK${`>)>skesT2ckRI0g9>=^>#6WubVl$hEQlcP_IJX$lm>hYSN15s%YtW2{ffUXc zBBm`MwA|_1)2<8%U4ZCHh4Z@vk&w(h6SS+r%c75U%|;{i)F0@qjSFOYp8VQ99{D;6 z1}J4R*`pY`*RihYhq;V4VOhnp6Xi!5yZ7>49bBh}e&~$}2u*oo6!ABXbQ#N^YeNTw zotvEN5NnY;ZEA3aM|r17UAvzOs@zlqd!!Kkc5mYb2Sq#u56y5_vY+~lLVPG@yxk&4 zJfq%-8Hl%^(e)DQ^)8>?tMz0W#~`n)Z0J;5NfWA+-g2g=A0tT?r%ooZ^O&tef9esQ zV{kVLL}c;M!54pGvBg^8HkiBXb{!HuFH#`g2-lwGSpqu%tGYNrO5k}=DqU=)jk*eR z@_CXcP+Y`0$jjeQF)Nmn?6^@#R+a@cT6OQdbYe162t)xD-XZv|hwf)irR31qD_$q$ z;W&AWOHVgSOkC$IJrd@h>hkiDBbs4$PY)1i#Nt^J!G;_htIGxS)Fk9MvE>|Tr&OFC zsf)?cy|rPCDmH+L4DoG^%CZTD>fHZcA1L%K^Z7LO@ro>swL;r8x*+kSNmie@v11lU zmvObzA>aAy{H#H@B+3Pmc(m{Wh#0CY_BW5c1K)o@+OAL%dI3zCBF+vB!jL-5JUo~5 z;!m9O&ep|e}D#iN+$`d@4`u3hlycE5Pd9YiD-S+r7LOH8(#*2?PYh$H)ZoK|*b2i|=p3jUbc z-5E{kP_cTZ2x|g~o&b4sDo1GbA;$e&d4pa_C<-BGa29kg_PO4yf7A%&PMS=!)5qxk z1rR<}DGs;z&I$44L!mtTSFJ0#x`mYde#-T))o~ww3YEDFL>7ATc7N=*_K^4@On0Uj zzj4&~+3KxhKmW3^Q=cJW%c&?6soM~G;f$sLdgiAgTU(SwB`Qo}3vAA6UI1QRg0zys z6Vv|M;>d>ets?B1P~VH1tF_5Z8YMnpP9>QSh$CkHio*ldvGHNi#o|JSeW73Ox4fG- zi)Z5zzvC%SpB8Yif%FB?m|8kzJ=SSCnHBi_9(3J|Aond{yaZnSdZzJVAvXQM2SaDX z2Ohwx;HTwCL?pu-Euaw4+D7c&g!Y+D0TOr7aO;18X zd?#xK)?vA|!om3XE+dIlT?Q+!|5Yc@zsb28)2ok*k`mtAl@=ArfU3{mpI~w=KOmMx34u}xCxHrGJs>FW$_-_xIZV$wk?72&;)Lt;# z(+0xh!ObDypiYjc0VVQ`4h}ZPU$AFaAAWiepUEd~I+{t8|&S?#zwqf@_jhHDAK z$m(ND7O!;+OIEm`m}joGK3P#&zJ-pf5{vfPLsu>xfCtpc>T&dLu%vtjw^}zAybyt-efkMNhJ@*mnDC$0XPYxqF=ooPNH@@{?QpEgy^&Ir|Y` zEoNm_PEIGYj`=cN)N+Y--N|nkUGO6KXzi*{bc`lcJaHOaOfe2Pfn5}G6@4C3O38`e zgKhGW<6BwWDUAk>J0XFd{FM;e+$|Jt?w=x=&p5cTT8{i=N%wuBf`|(k%GqDuY2hIY zeJ6=~_|>D#$$aOBR{5_(3x416WcLyWj`0fG2Rx5rINIFpuVe97OX8ChwzBNMTpdi$ z1Fr-&teoUlfg_OxC6b*VJO#fD0bFPx|1?$B8Df`>x8kQ%yJ-|Vxv>KU8`8lpnv}1= zRhQsWDOXO_pt`+cc>EK_BX7lx%&qg;D`Eb zhBC*BX1Pul|H7T%8}Q=SD=y}biYLkHM#G%bRGlA<@5~$VI_oaNjGGw7vEb9Yp8KAg z%@4AVHDq*Q5Bb%%i0ZwAlQApZ)#H#*uu}v63qZ@9ckdoi@0n1r!IZnVVZdc<#=oMs zI6mYc0eXETFdKh>m;X z_jJrR1Yzr1l$U46U{HyXI}T*46XT!)9026sJ7`X$Y8oS5dhL#D$(lZ!>Y|Ub0^Y|; zL{ZvcJ@~brG7eQrbd7)5=KWKd;%^ZfsPofGQ<&+L>sLyg(vE_6fn+QPUALKs*u(z6 z0y?`{>(>Z0Kjzfp;2H?n`@Jx}lHve5~i0xUzR94_*gwjOJwk(AOCUA~rhl zcAW&%S%dQ}V-D(dY# zFzV|Y8wcH$zyW!$dSZ?0$jBOA1fRl%43kJ-3s?9L7wjYauRr9?y|E?ey9QAlM&ca0 z)`J%Uyfs2eJ{Ow1__=rt4CCY99_UUa$hQ5&mD*h5NyR;SC#Npk1xWZq)P+tLC~xI% zL;KUnn6AJ-{_@dxOz(DtDrK|rC+XF@`YH0@E}|-u(PjPm4vbxq5Tt|U#>P8%P#LTD zt<_5Fp>;#noVatZd6Gx&j07U@B+*aI#-xEwPi2S7CfBb@$WF9 zIo@4eR$?w>|9S(ZsX&ufcAkGkR2Aux$rdvG=2k>P9;N?h97+jOHd-e7Gd^)Y)Uo~3eyO5FL~<0i^&lX$F-;}Btx}Z zJf*`I0H%cs7&Qj>!vgApxvIdl)$&YN z+1vW3t#$7lVXrHB+~6FCAdP3F#On$@aM2_eEnI3>tzU=e7Al_`kVLD>)lzt&tt@!ryolPRg|>MAPpJ5Wen_V1R

      Cay3tSG~Gl+lS*>nE;D5@akpLY1iQ@(1z zowWyzcmL_GOGIuSUpliiBehICga+QCbS^Z~3IUwD2d_tp;NSWRjB9OLL=XPp>Wds} z4j%L~8*vVD>>2=&?(ODu1l$_;`+@yn5Q@2-6xGw2&V|O~jkf6QawWYA$JC4XX+}Vo zfwKYorbvje$OPT_c%WkuSVmm_J8qBg-&?~pG4$vzw_~QGcT5aZs{EGU3%}grf1Tng zCz)H*m^!gBi?hRYwL0tgD{4gQ#4A+Yjo>T!=(JCQMP7x%P7V>mr_54T3oGm0sRRA_C4+MsM1ZM^5md=Cq|Bor;k4t(n#17$$#KKQkH!}Rnnt`uDS_fZZjc>oAoNE7%Hm*Gy3T}&^nel!^B@xDWHz^s9F&Z+-WB9zTyi3Sq zl1DQPLKyn=;*Ry*b=SIU-G9zrXP>=)d#|&8 z`>cKbIH#_CRW9!IzyTK<$EzhUQszxt4%fh0?zYNexiycjk zmDtES#w3Rzd*wA*z31z8fB^?mtxbSspwhRmmyIsaMA@;u zQ{9mcKrkkFFO5kjwZ--7%BNILh#83S1eV&XGq}flP9(X9L~EQY3Y8q*0k5EE=;E@?E))v#T`==54wy^3BShtP||$N#9v(SaWk!FCj(|kNXfCmhI?}n%J31i=vSR z^jeoBmE)%w1Ct-RYA~fioku88ip#;KnG&^kH1*VkA=eD$I(X^{8g|bP_;jJ@?8P<` z0#Wz|OEj1LP1};yY$CSz8R`iVMyPX8_>{QNw2RL3H?*p(w%4L^0se z$Bh2v+Qn2zhT(?MoyVpQ4l0$h`B!u!3QJBhP9&fO%xx64_Jt0pfhZpW=KB*5r@=Cm zY?{)M$**4Yz3(;oZ?#+P0EE8+-Bsx{(|vM(b?D7cG3KEQRB9eN*Tvsia|&zrH6#Ql zOXe>gSV+$K(q4Ej`}1HJot;%Su%$*(& z2v1CWc|Ak^c0wV&IJ-^j1SeWvyS6G>v0tir^jfw|1#4#YLry$DF2aIEN6mB6x} zV!@SY4zU<>EtitNLc?RI2Ww%DfU81_MjouB>ZX85KH?Se22#{H6e-q{dX2 zdVTh@mm)eUJq8~tLsrqersTspcL+p>~ z`od%jH~d-vXRBDfFDL(Amr7~Ms)+-aBVvU=<)R#`SBH)6i9ccW#GPmFfLp0fTT#|| zIiu3pWWMp1`+3D9vTsc-#%CENN)tO#X}Tf}h5i*1R@ z-@e0{dChr$-a1I~p~ynCEI}A!553On75a+#+E!l}CefnK%kC+)JUPrV->c*1k}g(~TmL%{?|)Qhak+$egW9L_??BMEH=ow5J#j-((;{fYnYt zA_zWepg9jNOZiRD6tmvZyouwNUbN4aciQ6wl$7^UH@9Dfw#-k_;bs@a<5l@=nW&LQ z8yn18Th=$|)4Mh;JpxzZ53^9eW9D0{TTHWzkX=xT``Y!NXtbHFbnj7DIa1y&9hI=N z

      Left Frame

      +

      This page is the left frame referenced by +this page. + + diff --git a/engine/testdata/selftest/FramesSelfTestCase/htdocs/noframe.html b/engine/testdata/selftest/FramesSelfTestCase/htdocs/noframe.html new file mode 100644 index 00000000..34c0f0b3 --- /dev/null +++ b/engine/testdata/selftest/FramesSelfTestCase/htdocs/noframe.html @@ -0,0 +1,11 @@ + + + No Frame + + +

      No Frame

      +

      This page is the no frame referenced by +this page when frames +don't work. + + diff --git a/engine/testdata/selftest/FramesSelfTestCase/htdocs/rightframe.html b/engine/testdata/selftest/FramesSelfTestCase/htdocs/rightframe.html new file mode 100644 index 00000000..28e73d7e --- /dev/null +++ b/engine/testdata/selftest/FramesSelfTestCase/htdocs/rightframe.html @@ -0,0 +1,10 @@ + + + Right Frame + + +

      Right Frame

      +

      This page is the left frame referenced by +this page. + + diff --git a/engine/testdata/selftest/FramesSelfTestCase/htdocs/topframe.html b/engine/testdata/selftest/FramesSelfTestCase/htdocs/topframe.html new file mode 100644 index 00000000..f3ef3999 --- /dev/null +++ b/engine/testdata/selftest/FramesSelfTestCase/htdocs/topframe.html @@ -0,0 +1,10 @@ + + + Top Frame + + +

      Top Frame

      +

      This page is the top frame referenced by +this page. + + diff --git a/engine/testdata/selftest/HttpAuthSelfTest/htdocs/basic/index.html b/engine/testdata/selftest/HttpAuthSelfTest/htdocs/basic/index.html new file mode 100644 index 00000000..db7fabf2 --- /dev/null +++ b/engine/testdata/selftest/HttpAuthSelfTest/htdocs/basic/index.html @@ -0,0 +1,7 @@ +index + +Link 1 + + Link 2 + + Link 3 diff --git a/engine/testdata/selftest/HttpAuthSelfTest/htdocs/basic/link1.html b/engine/testdata/selftest/HttpAuthSelfTest/htdocs/basic/link1.html new file mode 100644 index 00000000..e9521a25 --- /dev/null +++ b/engine/testdata/selftest/HttpAuthSelfTest/htdocs/basic/link1.html @@ -0,0 +1,7 @@ +index + +Link 1 + + Link 2 + + Link 3 \ No newline at end of file diff --git a/engine/testdata/selftest/HttpAuthSelfTest/htdocs/basic/link2.html b/engine/testdata/selftest/HttpAuthSelfTest/htdocs/basic/link2.html new file mode 100644 index 00000000..e9521a25 --- /dev/null +++ b/engine/testdata/selftest/HttpAuthSelfTest/htdocs/basic/link2.html @@ -0,0 +1,7 @@ +index + +Link 1 + + Link 2 + + Link 3 \ No newline at end of file diff --git a/engine/testdata/selftest/HttpAuthSelfTest/htdocs/basic/link3.html b/engine/testdata/selftest/HttpAuthSelfTest/htdocs/basic/link3.html new file mode 100644 index 00000000..e9521a25 --- /dev/null +++ b/engine/testdata/selftest/HttpAuthSelfTest/htdocs/basic/link3.html @@ -0,0 +1,7 @@ +index + +Link 1 + + Link 2 + + Link 3 \ No newline at end of file diff --git a/engine/testdata/selftest/HttpAuthSelfTest/htdocs/failure.html b/engine/testdata/selftest/HttpAuthSelfTest/htdocs/failure.html new file mode 100644 index 00000000..891cbb5a --- /dev/null +++ b/engine/testdata/selftest/HttpAuthSelfTest/htdocs/failure.html @@ -0,0 +1,9 @@ +index + +Link 1 + + Link 2 + + Link 3 + +Secure \ No newline at end of file diff --git a/engine/testdata/selftest/HttpAuthSelfTest/htdocs/index.html b/engine/testdata/selftest/HttpAuthSelfTest/htdocs/index.html new file mode 100644 index 00000000..891cbb5a --- /dev/null +++ b/engine/testdata/selftest/HttpAuthSelfTest/htdocs/index.html @@ -0,0 +1,9 @@ +index + +Link 1 + + Link 2 + + Link 3 + +Secure \ No newline at end of file diff --git a/engine/testdata/selftest/HttpAuthSelfTest/htdocs/link1.html b/engine/testdata/selftest/HttpAuthSelfTest/htdocs/link1.html new file mode 100644 index 00000000..e9521a25 --- /dev/null +++ b/engine/testdata/selftest/HttpAuthSelfTest/htdocs/link1.html @@ -0,0 +1,7 @@ +index + +Link 1 + + Link 2 + + Link 3 \ No newline at end of file diff --git a/engine/testdata/selftest/HttpAuthSelfTest/htdocs/link2.html b/engine/testdata/selftest/HttpAuthSelfTest/htdocs/link2.html new file mode 100644 index 00000000..e9521a25 --- /dev/null +++ b/engine/testdata/selftest/HttpAuthSelfTest/htdocs/link2.html @@ -0,0 +1,7 @@ +index + +Link 1 + + Link 2 + + Link 3 \ No newline at end of file diff --git a/engine/testdata/selftest/HttpAuthSelfTest/htdocs/link3.html b/engine/testdata/selftest/HttpAuthSelfTest/htdocs/link3.html new file mode 100644 index 00000000..e9521a25 --- /dev/null +++ b/engine/testdata/selftest/HttpAuthSelfTest/htdocs/link3.html @@ -0,0 +1,7 @@ +index + +Link 1 + + Link 2 + + Link 3 \ No newline at end of file diff --git a/engine/testdata/selftest/HttpAuthSelfTest/htdocs/success.html b/engine/testdata/selftest/HttpAuthSelfTest/htdocs/success.html new file mode 100644 index 00000000..891cbb5a --- /dev/null +++ b/engine/testdata/selftest/HttpAuthSelfTest/htdocs/success.html @@ -0,0 +1,9 @@ +index + +Link 1 + + Link 2 + + Link 3 + +Secure \ No newline at end of file diff --git a/engine/testdata/selftest/MaxLinkHopsSelfTest/htdocs/1.html b/engine/testdata/selftest/MaxLinkHopsSelfTest/htdocs/1.html new file mode 100644 index 00000000..3ec4d8c2 --- /dev/null +++ b/engine/testdata/selftest/MaxLinkHopsSelfTest/htdocs/1.html @@ -0,0 +1 @@ +link \ No newline at end of file diff --git a/engine/testdata/selftest/MaxLinkHopsSelfTest/htdocs/2.html b/engine/testdata/selftest/MaxLinkHopsSelfTest/htdocs/2.html new file mode 100644 index 00000000..6b36601e --- /dev/null +++ b/engine/testdata/selftest/MaxLinkHopsSelfTest/htdocs/2.html @@ -0,0 +1 @@ +link \ No newline at end of file diff --git a/engine/testdata/selftest/MaxLinkHopsSelfTest/htdocs/3.html b/engine/testdata/selftest/MaxLinkHopsSelfTest/htdocs/3.html new file mode 100644 index 00000000..2b0f0e9b --- /dev/null +++ b/engine/testdata/selftest/MaxLinkHopsSelfTest/htdocs/3.html @@ -0,0 +1 @@ +link \ No newline at end of file diff --git a/engine/testdata/selftest/MaxLinkHopsSelfTest/htdocs/4.html b/engine/testdata/selftest/MaxLinkHopsSelfTest/htdocs/4.html new file mode 100644 index 00000000..03884609 --- /dev/null +++ b/engine/testdata/selftest/MaxLinkHopsSelfTest/htdocs/4.html @@ -0,0 +1 @@ +link \ No newline at end of file diff --git a/engine/testdata/selftest/MaxLinkHopsSelfTest/htdocs/5.html b/engine/testdata/selftest/MaxLinkHopsSelfTest/htdocs/5.html new file mode 100644 index 00000000..ee199b4b --- /dev/null +++ b/engine/testdata/selftest/MaxLinkHopsSelfTest/htdocs/5.html @@ -0,0 +1 @@ +link \ No newline at end of file diff --git a/engine/testdata/selftest/MaxLinkHopsSelfTest/htdocs/index.html b/engine/testdata/selftest/MaxLinkHopsSelfTest/htdocs/index.html new file mode 100644 index 00000000..ee199b4b --- /dev/null +++ b/engine/testdata/selftest/MaxLinkHopsSelfTest/htdocs/index.html @@ -0,0 +1 @@ +link \ No newline at end of file diff --git a/engine/testdata/selftest/Precedence1SelfTest/htdocs/five/a.html b/engine/testdata/selftest/Precedence1SelfTest/htdocs/five/a.html new file mode 100644 index 00000000..c64b66a3 --- /dev/null +++ b/engine/testdata/selftest/Precedence1SelfTest/htdocs/five/a.html @@ -0,0 +1,4 @@ + +link #1 +link #2 +link #3 diff --git a/engine/testdata/selftest/Precedence1SelfTest/htdocs/five/b.html b/engine/testdata/selftest/Precedence1SelfTest/htdocs/five/b.html new file mode 100644 index 00000000..80d63a5a --- /dev/null +++ b/engine/testdata/selftest/Precedence1SelfTest/htdocs/five/b.html @@ -0,0 +1,4 @@ + +link #1 +link #2 +link #3 diff --git a/engine/testdata/selftest/Precedence1SelfTest/htdocs/one/a.html b/engine/testdata/selftest/Precedence1SelfTest/htdocs/one/a.html new file mode 100644 index 00000000..c64b66a3 --- /dev/null +++ b/engine/testdata/selftest/Precedence1SelfTest/htdocs/one/a.html @@ -0,0 +1,4 @@ + +link #1 +link #2 +link #3 diff --git a/engine/testdata/selftest/Precedence1SelfTest/htdocs/one/b.html b/engine/testdata/selftest/Precedence1SelfTest/htdocs/one/b.html new file mode 100644 index 00000000..80d63a5a --- /dev/null +++ b/engine/testdata/selftest/Precedence1SelfTest/htdocs/one/b.html @@ -0,0 +1,4 @@ + +link #1 +link #2 +link #3 diff --git a/engine/testdata/selftest/Precedence1SelfTest/htdocs/seed.html b/engine/testdata/selftest/Precedence1SelfTest/htdocs/seed.html new file mode 100644 index 00000000..20fdcd20 --- /dev/null +++ b/engine/testdata/selftest/Precedence1SelfTest/htdocs/seed.html @@ -0,0 +1,4 @@ + +link #1 +link #2 +link #3 diff --git a/engine/testdata/selftest/Precedence1SelfTest/htdocs/ten/a.html b/engine/testdata/selftest/Precedence1SelfTest/htdocs/ten/a.html new file mode 100644 index 00000000..c64b66a3 --- /dev/null +++ b/engine/testdata/selftest/Precedence1SelfTest/htdocs/ten/a.html @@ -0,0 +1,4 @@ + +link #1 +link #2 +link #3 diff --git a/engine/testdata/selftest/Precedence1SelfTest/htdocs/ten/b.html b/engine/testdata/selftest/Precedence1SelfTest/htdocs/ten/b.html new file mode 100644 index 00000000..80d63a5a --- /dev/null +++ b/engine/testdata/selftest/Precedence1SelfTest/htdocs/ten/b.html @@ -0,0 +1,4 @@ + +link #1 +link #2 +link #3 diff --git a/engine/testdata/selftest/Precedence2SelfTest/htdocs/five/a.html b/engine/testdata/selftest/Precedence2SelfTest/htdocs/five/a.html new file mode 100644 index 00000000..c64b66a3 --- /dev/null +++ b/engine/testdata/selftest/Precedence2SelfTest/htdocs/five/a.html @@ -0,0 +1,4 @@ + +link #1 +link #2 +link #3 diff --git a/engine/testdata/selftest/Precedence2SelfTest/htdocs/five/b.html b/engine/testdata/selftest/Precedence2SelfTest/htdocs/five/b.html new file mode 100644 index 00000000..80d63a5a --- /dev/null +++ b/engine/testdata/selftest/Precedence2SelfTest/htdocs/five/b.html @@ -0,0 +1,4 @@ + +link #1 +link #2 +link #3 diff --git a/engine/testdata/selftest/Precedence2SelfTest/htdocs/one/a.html b/engine/testdata/selftest/Precedence2SelfTest/htdocs/one/a.html new file mode 100644 index 00000000..c64b66a3 --- /dev/null +++ b/engine/testdata/selftest/Precedence2SelfTest/htdocs/one/a.html @@ -0,0 +1,4 @@ + +link #1 +link #2 +link #3 diff --git a/engine/testdata/selftest/Precedence2SelfTest/htdocs/one/b.html b/engine/testdata/selftest/Precedence2SelfTest/htdocs/one/b.html new file mode 100644 index 00000000..80d63a5a --- /dev/null +++ b/engine/testdata/selftest/Precedence2SelfTest/htdocs/one/b.html @@ -0,0 +1,4 @@ + +link #1 +link #2 +link #3 diff --git a/engine/testdata/selftest/Precedence2SelfTest/htdocs/seed.html b/engine/testdata/selftest/Precedence2SelfTest/htdocs/seed.html new file mode 100644 index 00000000..20fdcd20 --- /dev/null +++ b/engine/testdata/selftest/Precedence2SelfTest/htdocs/seed.html @@ -0,0 +1,4 @@ + +link #1 +link #2 +link #3 diff --git a/engine/testdata/selftest/Precedence2SelfTest/htdocs/ten/a.html b/engine/testdata/selftest/Precedence2SelfTest/htdocs/ten/a.html new file mode 100644 index 00000000..c64b66a3 --- /dev/null +++ b/engine/testdata/selftest/Precedence2SelfTest/htdocs/ten/a.html @@ -0,0 +1,4 @@ + +link #1 +link #2 +link #3 diff --git a/engine/testdata/selftest/Precedence2SelfTest/htdocs/ten/b.html b/engine/testdata/selftest/Precedence2SelfTest/htdocs/ten/b.html new file mode 100644 index 00000000..80d63a5a --- /dev/null +++ b/engine/testdata/selftest/Precedence2SelfTest/htdocs/ten/b.html @@ -0,0 +1,4 @@ + +link #1 +link #2 +link #3 diff --git a/engine/testdata/selftest/Precedence2SelfTest/profile/rank.txt b/engine/testdata/selftest/Precedence2SelfTest/profile/rank.txt new file mode 100644 index 00000000..9f24ccb1 --- /dev/null +++ b/engine/testdata/selftest/Precedence2SelfTest/profile/rank.txt @@ -0,0 +1,10 @@ +http://127.0.0.1:7777/seed.html 1 +http://127.0.0.1:7777/one/a.html 1 +http://127.0.0.1:7777/one/b.html 1 +http://127.0.0.1:7777/one/c.html 1 +http://127.0.0.1:7777/five/a.html 5 +http://127.0.0.1:7777/five/b.html 5 +http://127.0.0.1:7777/five/c.html 5 +http://127.0.0.1:7777/ten/a.html 10 +http://127.0.0.1:7777/ten/b.html 10 +http://127.0.0.1:7777/ten/c.html 10 diff --git a/engine/testdata/selftest/Precedence3SelfTest/htdocs/A.html b/engine/testdata/selftest/Precedence3SelfTest/htdocs/A.html new file mode 100644 index 00000000..5709f6f9 --- /dev/null +++ b/engine/testdata/selftest/Precedence3SelfTest/htdocs/A.html @@ -0,0 +1,3 @@ +keyword + +leaf diff --git a/engine/testdata/selftest/Precedence3SelfTest/htdocs/B.html b/engine/testdata/selftest/Precedence3SelfTest/htdocs/B.html new file mode 100644 index 00000000..ac0ea831 --- /dev/null +++ b/engine/testdata/selftest/Precedence3SelfTest/htdocs/B.html @@ -0,0 +1,4 @@ +keyword + +Link 1 +Link 2 diff --git a/engine/testdata/selftest/Precedence3SelfTest/htdocs/C.html b/engine/testdata/selftest/Precedence3SelfTest/htdocs/C.html new file mode 100644 index 00000000..9a07dce5 --- /dev/null +++ b/engine/testdata/selftest/Precedence3SelfTest/htdocs/C.html @@ -0,0 +1 @@ +leaf diff --git a/engine/testdata/selftest/Precedence3SelfTest/htdocs/D.html b/engine/testdata/selftest/Precedence3SelfTest/htdocs/D.html new file mode 100644 index 00000000..4d48e778 --- /dev/null +++ b/engine/testdata/selftest/Precedence3SelfTest/htdocs/D.html @@ -0,0 +1,4 @@ +keyword + +Link 1 +Link 2 diff --git a/engine/testdata/selftest/Precedence3SelfTest/htdocs/E.html b/engine/testdata/selftest/Precedence3SelfTest/htdocs/E.html new file mode 100644 index 00000000..5709f6f9 --- /dev/null +++ b/engine/testdata/selftest/Precedence3SelfTest/htdocs/E.html @@ -0,0 +1,3 @@ +keyword + +leaf diff --git a/engine/testdata/selftest/Precedence3SelfTest/htdocs/F.html b/engine/testdata/selftest/Precedence3SelfTest/htdocs/F.html new file mode 100644 index 00000000..09ca0527 --- /dev/null +++ b/engine/testdata/selftest/Precedence3SelfTest/htdocs/F.html @@ -0,0 +1,3 @@ + +Link 1 +Link 2 diff --git a/engine/testdata/selftest/Precedence3SelfTest/htdocs/G.html b/engine/testdata/selftest/Precedence3SelfTest/htdocs/G.html new file mode 100644 index 00000000..c47d85d3 --- /dev/null +++ b/engine/testdata/selftest/Precedence3SelfTest/htdocs/G.html @@ -0,0 +1,2 @@ + +leaf diff --git a/engine/testdata/selftest/Precedence3SelfTest/htdocs/H.html b/engine/testdata/selftest/Precedence3SelfTest/htdocs/H.html new file mode 100644 index 00000000..67eb3971 --- /dev/null +++ b/engine/testdata/selftest/Precedence3SelfTest/htdocs/H.html @@ -0,0 +1,4 @@ +keyword + +Link 1 +Link 2 diff --git a/engine/testdata/selftest/Precedence3SelfTest/htdocs/I.html b/engine/testdata/selftest/Precedence3SelfTest/htdocs/I.html new file mode 100644 index 00000000..5709f6f9 --- /dev/null +++ b/engine/testdata/selftest/Precedence3SelfTest/htdocs/I.html @@ -0,0 +1,3 @@ +keyword + +leaf diff --git a/engine/testdata/selftest/Precedence3SelfTest/htdocs/J.html b/engine/testdata/selftest/Precedence3SelfTest/htdocs/J.html new file mode 100644 index 00000000..41ce9be8 --- /dev/null +++ b/engine/testdata/selftest/Precedence3SelfTest/htdocs/J.html @@ -0,0 +1,4 @@ +keyword + +Link 1 +Link 2 diff --git a/engine/testdata/selftest/Precedence3SelfTest/htdocs/K.html b/engine/testdata/selftest/Precedence3SelfTest/htdocs/K.html new file mode 100644 index 00000000..c47d85d3 --- /dev/null +++ b/engine/testdata/selftest/Precedence3SelfTest/htdocs/K.html @@ -0,0 +1,2 @@ + +leaf diff --git a/engine/testdata/selftest/Precedence3SelfTest/htdocs/L.html b/engine/testdata/selftest/Precedence3SelfTest/htdocs/L.html new file mode 100644 index 00000000..b2295284 --- /dev/null +++ b/engine/testdata/selftest/Precedence3SelfTest/htdocs/L.html @@ -0,0 +1,2 @@ +Link 1 +Link 2 diff --git a/engine/testdata/selftest/Precedence3SelfTest/htdocs/M.html b/engine/testdata/selftest/Precedence3SelfTest/htdocs/M.html new file mode 100644 index 00000000..5709f6f9 --- /dev/null +++ b/engine/testdata/selftest/Precedence3SelfTest/htdocs/M.html @@ -0,0 +1,3 @@ +keyword + +leaf diff --git a/engine/testdata/selftest/Precedence3SelfTest/htdocs/N.html b/engine/testdata/selftest/Precedence3SelfTest/htdocs/N.html new file mode 100644 index 00000000..0cb681f8 --- /dev/null +++ b/engine/testdata/selftest/Precedence3SelfTest/htdocs/N.html @@ -0,0 +1,3 @@ + +Link 1 +Link 2 diff --git a/engine/testdata/selftest/Precedence3SelfTest/htdocs/O.html b/engine/testdata/selftest/Precedence3SelfTest/htdocs/O.html new file mode 100644 index 00000000..c47d85d3 --- /dev/null +++ b/engine/testdata/selftest/Precedence3SelfTest/htdocs/O.html @@ -0,0 +1,2 @@ + +leaf diff --git a/engine/testdata/selftest/Precedence4SelfTest/htdocs/five/a.html b/engine/testdata/selftest/Precedence4SelfTest/htdocs/five/a.html new file mode 100644 index 00000000..c64b66a3 --- /dev/null +++ b/engine/testdata/selftest/Precedence4SelfTest/htdocs/five/a.html @@ -0,0 +1,4 @@ + +link #1 +link #2 +link #3 diff --git a/engine/testdata/selftest/Precedence4SelfTest/htdocs/five/b.html b/engine/testdata/selftest/Precedence4SelfTest/htdocs/five/b.html new file mode 100644 index 00000000..80d63a5a --- /dev/null +++ b/engine/testdata/selftest/Precedence4SelfTest/htdocs/five/b.html @@ -0,0 +1,4 @@ + +link #1 +link #2 +link #3 diff --git a/engine/testdata/selftest/Precedence4SelfTest/htdocs/one/a.html b/engine/testdata/selftest/Precedence4SelfTest/htdocs/one/a.html new file mode 100644 index 00000000..c64b66a3 --- /dev/null +++ b/engine/testdata/selftest/Precedence4SelfTest/htdocs/one/a.html @@ -0,0 +1,4 @@ + +link #1 +link #2 +link #3 diff --git a/engine/testdata/selftest/Precedence4SelfTest/htdocs/one/b.html b/engine/testdata/selftest/Precedence4SelfTest/htdocs/one/b.html new file mode 100644 index 00000000..80d63a5a --- /dev/null +++ b/engine/testdata/selftest/Precedence4SelfTest/htdocs/one/b.html @@ -0,0 +1,4 @@ + +link #1 +link #2 +link #3 diff --git a/engine/testdata/selftest/Precedence4SelfTest/htdocs/seed.html b/engine/testdata/selftest/Precedence4SelfTest/htdocs/seed.html new file mode 100644 index 00000000..20fdcd20 --- /dev/null +++ b/engine/testdata/selftest/Precedence4SelfTest/htdocs/seed.html @@ -0,0 +1,4 @@ + +link #1 +link #2 +link #3 diff --git a/engine/testdata/selftest/Precedence4SelfTest/htdocs/ten/a.html b/engine/testdata/selftest/Precedence4SelfTest/htdocs/ten/a.html new file mode 100644 index 00000000..c64b66a3 --- /dev/null +++ b/engine/testdata/selftest/Precedence4SelfTest/htdocs/ten/a.html @@ -0,0 +1,4 @@ + +link #1 +link #2 +link #3 diff --git a/engine/testdata/selftest/Precedence4SelfTest/htdocs/ten/b.html b/engine/testdata/selftest/Precedence4SelfTest/htdocs/ten/b.html new file mode 100644 index 00000000..80d63a5a --- /dev/null +++ b/engine/testdata/selftest/Precedence4SelfTest/htdocs/ten/b.html @@ -0,0 +1,4 @@ + +link #1 +link #2 +link #3 diff --git a/engine/testdata/selftest/SimpleSelfTest/htdocs/index.html b/engine/testdata/selftest/SimpleSelfTest/htdocs/index.html new file mode 100644 index 00000000..db7fabf2 --- /dev/null +++ b/engine/testdata/selftest/SimpleSelfTest/htdocs/index.html @@ -0,0 +1,7 @@ +index + +Link 1 + + Link 2 + + Link 3 diff --git a/engine/testdata/selftest/SimpleSelfTest/htdocs/link1.html b/engine/testdata/selftest/SimpleSelfTest/htdocs/link1.html new file mode 100644 index 00000000..e9521a25 --- /dev/null +++ b/engine/testdata/selftest/SimpleSelfTest/htdocs/link1.html @@ -0,0 +1,7 @@ +index + +Link 1 + + Link 2 + + Link 3 \ No newline at end of file diff --git a/engine/testdata/selftest/SimpleSelfTest/htdocs/link2.html b/engine/testdata/selftest/SimpleSelfTest/htdocs/link2.html new file mode 100644 index 00000000..e9521a25 --- /dev/null +++ b/engine/testdata/selftest/SimpleSelfTest/htdocs/link2.html @@ -0,0 +1,7 @@ +index + +Link 1 + + Link 2 + + Link 3 \ No newline at end of file diff --git a/engine/testdata/selftest/SimpleSelfTest/htdocs/link3.html b/engine/testdata/selftest/SimpleSelfTest/htdocs/link3.html new file mode 100644 index 00000000..e9521a25 --- /dev/null +++ b/engine/testdata/selftest/SimpleSelfTest/htdocs/link3.html @@ -0,0 +1,7 @@ +index + +Link 1 + + Link 2 + + Link 3 \ No newline at end of file diff --git a/engine/testdata/selftest/conf/heritrix.properties b/engine/testdata/selftest/conf/heritrix.properties new file mode 100644 index 00000000..d2a74f8c --- /dev/null +++ b/engine/testdata/selftest/conf/heritrix.properties @@ -0,0 +1,15 @@ +handlers=java.util.logging.ConsoleHandler +java.util.logging.ConsoleHandler.level=ALL + +# Default global logging level: only warnings or higher +.level=ALL + +# currently necessary (?) for standard logs to work +crawl.level=INFO +runtime-errors.level=INFO +uri-errors.level=INFO +progress-statistics.level=INFO +recover.level=INFO + +# HttpClient is too chatty... only want to hear about severe problems +org.apache.commons.httpclient.level=SEVERE \ No newline at end of file diff --git a/engine/testdata/selftest/conf/selftest-crawler-beans.cxml b/engine/testdata/selftest/conf/selftest-crawler-beans.cxml new file mode 100644 index 00000000..a5efcb52 --- /dev/null +++ b/engine/testdata/selftest/conf/selftest-crawler-beans.cxml @@ -0,0 +1,181 @@ + + + + + + + + + +metadata.operatorContactUrl=@@URL_VALUE@@ +seeds.textSource.value=@@SEEDS_VALUE@@ +# only self-crawling, so go fast +frontier.maxDelayMs=50 +@@MORE_PROPERTIES@@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/modules/.classpath b/modules/.classpath new file mode 100644 index 00000000..d076610b --- /dev/null +++ b/modules/.classpath @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/modules/.project b/modules/.project new file mode 100644 index 00000000..9e9c8e44 --- /dev/null +++ b/modules/.project @@ -0,0 +1,15 @@ + + modules + This project contains some of the configurable modules used within the +Heritrix application to crawl the web. The modules in this project can +be used in applications other than Heritrix, however. + + + + org.eclipse.jdt.core.javabuilder + + + + org.eclipse.jdt.core.javanature + + \ No newline at end of file diff --git a/modules/.settings/org.eclipse.jdt.core.prefs b/modules/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 00000000..9ec5bb48 --- /dev/null +++ b/modules/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,5 @@ +#Wed Apr 11 15:05:43 PDT 2007 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.5 +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.source=1.5 +org.eclipse.jdt.core.compiler.compliance=1.5 diff --git a/modules/pom.xml b/modules/pom.xml new file mode 100644 index 00000000..17d86f9a --- /dev/null +++ b/modules/pom.xml @@ -0,0 +1,78 @@ + + + + + org.archive + heritrix + 3.0.0-SNAPSHOT + + 4.0.0 + org.archive.heritrix + modules + jar + Heritrix: 'modules' subproject (reusable components) + + +This project contains some of the configurable modules used within the +Heritrix application to crawl the web. The modules in this project can +be used in applications other than Heritrix, however. + + + + + + + true + daily + warn + + + true + never + fail + + internetarchive + Internet Archive Maven Repository + http://builds.archive.org:8080/maven2 + default + + + + + + org.archive.heritrix + commons + ${project.version} + compile + + + org.beanshell + bsh + 2.0b4 + compile + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.5 + 1.5 + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + **/TestAll.java + + + + + + diff --git a/modules/src/main/java/org/archive/modules/BeanShellProcessor.java b/modules/src/main/java/org/archive/modules/BeanShellProcessor.java new file mode 100644 index 00000000..8c17bf04 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/BeanShellProcessor.java @@ -0,0 +1,176 @@ +/* BeanShellProcessor + * + * Created on Aug 4, 2006 + * + * Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.modules; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.archive.spring.ConfigPath; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.annotation.Required; +import org.springframework.context.ApplicationContext; + +import bsh.EvalError; +import bsh.Interpreter; + +/** + * A processor which runs a BeanShell script on the CrawlURI. + * + * Script source may be provided via a file + * local to the crawler. + * Script source should define + * a method with one argument, 'run(curi)'. Each processed CrawlURI is + * passed to this script method. + * + * Other variables available to the script include 'self' (this + * BeanShellProcessor instance) and 'controller' (the crawl's + * CrawlController instance). + * + * @author gojomo + * @version $Date$, $Revision$ + */ +public class BeanShellProcessor extends Processor { + + private static final long serialVersionUID = 3L; + + private static final Logger logger = + Logger.getLogger(BeanShellProcessor.class.getName()); + + + /** + * BeanShell script file. + */ + ConfigPath scriptFile = null; + public ConfigPath getScriptFile() { + return this.scriptFile; + } + @Required + public void setScriptFile(ConfigPath file) { + this.scriptFile = file; + } + + /** + * Whether each ToeThread should get its own independent script context, or + * they should share synchronized access to one context. Default is true, + * meaning each threads gets its own isolated context. + * + */ + protected boolean isolateThreads = true; + public boolean getIsolateThreads() { + return isolateThreads; + } + public void setIsolateThreads(boolean isolateThreads) { + this.isolateThreads = isolateThreads; + } + + ApplicationContext appCtx; + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + this.appCtx = applicationContext; + } + + protected ThreadLocal threadInterpreter; + protected Interpreter sharedInterpreter; + public Map sharedMap = Collections.synchronizedMap( + new HashMap()); + + /** + * Constructor. + */ + public BeanShellProcessor() { + super(); + } + + protected boolean shouldProcess(ProcessorURI curi) { + return true; + } + + @Override + protected synchronized void innerProcess(ProcessorURI curi) { + // depending on previous configuration, interpreter may + // be local to this thread or shared + Interpreter interpreter = getInterpreter(); + synchronized(interpreter) { + // synchronization is harmless for local thread interpreter, + // necessary for shared interpreter + try { + interpreter.set("curi",curi); + interpreter.eval("process(curi)"); + } catch (EvalError e) { + logger.log(Level.WARNING,"BeanShell error", e); + } + } + } + + /** + * Get the proper Interpreter instance -- either shared or local + * to this thread. + * @return Interpreter to use + */ + protected Interpreter getInterpreter() { + if(sharedInterpreter==null + && getIsolateThreads()) { + // initialize + sharedInterpreter = newInterpreter(); + } + if(sharedInterpreter!=null) { + return sharedInterpreter; + } + Interpreter interpreter = threadInterpreter.get(); + if(interpreter==null) { + interpreter = newInterpreter(); + threadInterpreter.set(interpreter); + } + return interpreter; + } + + /** + * Create a new Interpreter instance, preloaded with any supplied + * source code or source file and the variables 'self' (this + * BeanShellProcessor) and 'controller' (the CrawlController). + * + * @return the new Interpreter instance + */ + protected Interpreter newInterpreter() { + Interpreter interpreter = new Interpreter(); + try { + interpreter.set("self", this); + interpreter.set("context", appCtx); + + File file = getScriptFile().getFile(); + try { + interpreter.source(file.getPath()); + } catch (IOException e) { + logger.log(Level.SEVERE,"unable to read script file",e); + } + } catch (EvalError e) { + logger.log(Level.SEVERE,"error in source file",e); + } + + return interpreter; + } +} diff --git a/modules/src/main/java/org/archive/modules/CrawlMetadata.java b/modules/src/main/java/org/archive/modules/CrawlMetadata.java new file mode 100644 index 00000000..c858d1c8 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/CrawlMetadata.java @@ -0,0 +1,191 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.modules; + +import java.io.Serializable; +import java.util.regex.Matcher; + +import org.archive.modules.fetcher.UserAgentProvider; +import org.archive.modules.net.RobotsHonoringPolicy; +import org.archive.spring.BeanFieldsPatternValidator; +import org.archive.spring.HasKeyedProperties; +import org.archive.spring.HasValidator; +import org.archive.spring.KeyedProperties; +import org.archive.util.ArchiveUtils; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.validation.Validator; + +/** + * Basic crawl metadata, as consulted by functional modules and + * recorded in ARCs/WARCs. + * + * @contributor pjack + */ +public class CrawlMetadata +implements UserAgentProvider, + Serializable, + HasKeyedProperties, + HasValidator, + InitializingBean { + private static final long serialVersionUID = 1L; + + KeyedProperties kp = new KeyedProperties(); + public KeyedProperties getKeyedProperties() { + return kp; + } + + public RobotsHonoringPolicy getRobotsHonoringPolicy() { + return (RobotsHonoringPolicy) kp.get("robotsHonoringPolicy"); + } + @Autowired + public void setRobotsHonoringPolicy(RobotsHonoringPolicy policy) { + kp.put("robotsHonoringPolicy",policy); + } + + String operator = ""; + public String getOperator() { + return operator; + } + public void setOperator(String operatorName) { + this.operator = operatorName; + } + + String description = ""; + public String getDescription() { + return description; + } + public void setDescription(String description) { + this.description = description; + } + + { + setUserAgentTemplate("Mozilla/5.0 (compatible; heritrix/@VERSION@ +@OPERATOR_CONTACT_URL@)"); + } + public String getUserAgentTemplate() { + return (String) kp.get("userAgentTemplate"); + } + public void setUserAgentTemplate(String template) { + // TODO compile pattern outside method +// if(!template.matches("^.*\\+@OPERATOR_CONTACT_URL@.*$")) { +// throw new IllegalArgumentException("bad user-agent: "+template); +// } + kp.put("userAgentTemplate",template); + } + + { + setOperatorFrom(""); + } + public String getOperatorFrom() { + return (String) kp.get("operatorFrom"); + } + public void setOperatorFrom(String operatorFrom) { + // TODO compile pattern outside method +// if(!operatorFrom.matches("^(\\s*|\\S+@[-\\w]+\\.[-\\w\\.]+)$")) { +// throw new IllegalArgumentException("bad operatorFrom: "+operatorFrom); +// } + kp.put("operatorFrom",operatorFrom); + } + + { + // set default to illegal value + kp.put("operatorContactUrl","ENTER-A-CONTACT-HTTP-URL-FOR-CRAWL-OPERATOR"); + } + public String getOperatorContactUrl() { + return (String) kp.get("operatorContactUrl"); + } + public void setOperatorContactUrl(String operatorContactUrl) { + // TODO compile pattern outside method +// if(!operatorContactUrl.matches("^https?://.*$")) { +// throw new IllegalArgumentException("bad operatorContactUrl: "+operatorContactUrl); +// } + kp.put("operatorContactUrl",operatorContactUrl); + } + + + String audience = ""; + public String getAudience() { + return audience; + } + public void setAudience(String audience) { + this.audience = audience; + } + + String organization = ""; + public String getOrganization() { + return organization; + } + public void setOrganization(String organization) { + this.organization = organization; + } + + public String getUserAgent() { + String userAgent = getUserAgentTemplate(); + String contactURL = getOperatorContactUrl(); + userAgent = userAgent.replaceFirst("@OPERATOR_CONTACT_URL@", contactURL); + userAgent = userAgent.replaceFirst("@VERSION@", + Matcher.quoteReplacement(ArchiveUtils.VERSION)); + return userAgent; + } + + String jobName; + public String getJobName() { + return jobName; + } + public void setJobName(String jobName) { + this.jobName = jobName; + } + + public String getRobotsPolicyName() { + return getRobotsHonoringPolicy().getType().toString(); + } + + public String getFrom() { + return getOperatorFrom(); + } + + public void afterPropertiesSet() throws Exception { + // force revalidation, throwing exception if invalid + setOperatorContactUrl(getOperatorContactUrl()); + } + + static Validator VALIDATOR = new BeanFieldsPatternValidator( + CrawlMetadata.class, + "userAgentTemplate", + "^.*\\+@OPERATOR_CONTACT_URL@.*$", + "You must supply a userAgentTemplate value that includes " + + "the string \"@OPERATOR_CONTACT_URL@\" where your crawl" + + "contact URL will appear.", + + "operatorContactUrl", + "^https?://.*$", + "You must supply an HTTP(S) URL which will be included " + + "in your user-agent and should explain the purpose of your " + + "crawl and how to contact the crawl operator in the event " + + "of webmaster issues.", + + "operatorFrom", + "^(\\s*|\\S+@[-\\w]+\\.[-\\w\\.]+)|()$", + "If not blank, operatorFrom must be an email address."); + + public Validator getValidator() { + return VALIDATOR; + } +} diff --git a/modules/src/main/java/org/archive/modules/DefaultProcessorURI.java b/modules/src/main/java/org/archive/modules/DefaultProcessorURI.java new file mode 100644 index 00000000..3f4f878a --- /dev/null +++ b/modules/src/main/java/org/archive/modules/DefaultProcessorURI.java @@ -0,0 +1,410 @@ +package org.archive.modules; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.httpclient.HttpMethod; +import org.apache.commons.httpclient.URIException; +import org.archive.modules.credential.CredentialAvatar; +import org.archive.modules.extractor.Link; +import org.archive.modules.extractor.LinkContext; +import org.archive.modules.net.CrawlHost; +import org.archive.modules.net.CrawlServer; +import org.archive.modules.net.RobotsHonoringPolicy; +import org.archive.net.UURI; +import org.archive.util.Recorder; + + +public class DefaultProcessorURI +implements ProcessorURI { + + + /** + * An exception and an error message. Used for keeping list of errors. + */ + public static class ExceptionHolder { + + /** The exception. */ + final public Throwable exception; + + /** The error message. */ + final public String description; + + /** + * Constructor. + * + * @param e the exception + * @param desc the description + */ + public ExceptionHolder(Throwable e, String desc) { + this.exception = e; + this.description = desc; + } + } + + + final private Collection annotations = new ArrayList(); + final private Set avatars = + new HashSet(); + final private List nonFatalFailures = + new ArrayList(); + final private List outLinks = new ArrayList(); + final private List uriErrors + = new ArrayList(); + + private int discardedOutLinks; + + private String from; + private String userAgent; + + private long length; + private long size; + private String contentType; + + private long startTime; + private long endTime; + + private int fetchStatus; + private FetchType fetchType = FetchType.UNKNOWN; + private HttpMethod httpMethod; + + private String pathFromSeed; + + private UURI base; + + private Map data = new HashMap(); + + private Recorder recorder; + + private String resolvedName; + private RobotsHonoringPolicy robotsHonoringPolicy; + + private UURI uuri; + private UURI via; + private LinkContext viaContext; + + private boolean linkExtractionFinished; + private boolean location; + private boolean prereq; + private boolean seed; + + private boolean forceFetch; + + + public DefaultProcessorURI(UURI uuri, LinkContext context) { + this.uuri = uuri; + this.viaContext = context; + } + + + public void addUriError(URIException e, String uri) { + uriErrors.add(new ExceptionHolder(e, uri)); + } + + public List getUriErrors() { + return uriErrors; + } + + public boolean attachRfc2617Credential(String realm) { + // TODO Auto-generated method stub + return false; + } + + public boolean detachRfc2617Credential(String realm) { + // TODO Auto-generated method stub + return false; + } + + public Collection getAnnotations() { + return annotations; + } + + public UURI getBaseURI() { + return base; + } + + public long getContentLength() { + return length; + } + + + public void setContentLength(long len) { + this.length = len; + } + + public long getContentSize() { + return size; + } + + public String getContentType() { + return contentType; + } + + public CrawlHost getCrawlHost() { + // TODO Auto-generated method stub + return null; + } + + + public CrawlServer getCrawlServer(String serverKey) { + // TODO + return null; + } + + + public boolean hasCredentialAvatars() { + return getCredentialAvatars().size() > 0; + } + + + public Set getCredentialAvatars() { + return avatars; + } + + public String getDNSServerIPLabel() { + // TODO Auto-generated method stub + return null; + } + + public Map getData() { + return data; + } + + public long getFetchBeginTime() { + return startTime; + } + + public long getFetchCompletedTime() { + return endTime; + } + + public int getFetchStatus() { + return fetchStatus; + } + + public FetchType getFetchType() { + return fetchType; + } + + public String getFrom() { + return from; + } + + public Collection getNonFatalFailures() { + return nonFatalFailures; + } + + public List getOutLinks() { + return outLinks; + } + + public String getPathFromSeed() { + return pathFromSeed; + } + + public Recorder getRecorder() { + return recorder; + } + + public void setRecorder(Recorder r) { + this.recorder = r; + } + + public String getResolvedName() { + return resolvedName; // FIXME: CrawlHost + } + + public RobotsHonoringPolicy getRobotsHonoringPolicy() { + return robotsHonoringPolicy; + } + + public UURI getUURI() { + return uuri; + } + + public String getUserAgent() { + return userAgent; + } + + public void setUserAgent(String ua) { + this.userAgent = ua; + } + + public UURI getVia() { + return via; + } + + public LinkContext getViaContext() { + return viaContext; + } + + public boolean hasBeenLinkExtracted() { + return linkExtractionFinished; + } + + public boolean isLocation() { + return location; + } + + public boolean isPrerequisite() { + return prereq; + } + + public boolean isSeed() { + return seed; + } + + public boolean passedDNS() { + // TODO Auto-generated method stub + return false; + } + + public boolean populateCredentials(HttpMethod method) { + // TODO Auto-generated method stub + return false; + } + + public void promoteCredentials() { + // TODO Auto-generated method stub + + } + + public void requestCrawlPause() { + // TODO Auto-generated method stub + + } + + public void setBaseURI(UURI base) { + this.base = base; + } + + public void setContentDigest(String algorithm, byte[] digest) { + // TODO Auto-generated method stub + + } + + public void setContentSize(long size) { + this.size = size; + } + + public void setContentType(String mimeType) { + this.contentType = mimeType; + } + + public void setDNSServerIPLabel(String label) { + // TODO Auto-generated method stub + + } + + public void setError(String msg) { + // TODO Auto-generated method stub + } + + public void setFetchBeginTime(long time) { + startTime = time; + } + + public void setFetchCompletedTime(long time) { + endTime = time; + } + + public void setFetchStatus(int status) { + this.fetchStatus = status; + } + + public void setFetchType(FetchType type) { + if (type == null) { + throw new IllegalArgumentException("fetchType is non-null"); + } + this.fetchType = type; + } + + public void setHttpMethod(HttpMethod method) { + this.httpMethod = method; + } + + public void linkExtractorFinished() { + linkExtractionFinished = true; + } + + public void setPrerequisite(boolean prereq) { + this.prereq = prereq; + } + + public void setSeed(boolean seed) { + this.seed = seed; + } + + public void skipToPostProcessing() { + // TODO Auto-generated method stub + + } + + + public HttpMethod getHttpMethod() { + return this.httpMethod; + } + + + public int getFetchAttempts() { + return 1; // FIXME + } + + + public boolean containsDataKey(String k) { + return data.containsKey(k); + } + + + public byte[] getContentDigest() { + return null; // FIXME + } + + + public String getContentDigestSchemeString() { + // FIXME + return null; + } + + public void makeHeritable(String attr) { + // FIXME? -- maybe irrelevant + + } + + public boolean isSuccess() { + // FIXME? + return false; + } + + public Map getPersistentDataMap() { + // FIXME? + return null; + } + + public void addPersistentDataMapKey(String s) { + + } + + + public void incrementDiscardedOutLinks() { + discardedOutLinks++; + } + + + public boolean forceFetch() { + return forceFetch; + } + + public void setForceFetch(boolean b) { + this.forceFetch = b; + } + + public String getSourceTag() { + return ""; + } +} diff --git a/modules/src/main/java/org/archive/modules/ModuleAttributeConstants.java b/modules/src/main/java/org/archive/modules/ModuleAttributeConstants.java new file mode 100644 index 00000000..aa417dc6 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/ModuleAttributeConstants.java @@ -0,0 +1,40 @@ +/** + * + */ +package org.archive.modules; + + +/** + * @author pjack + * + */ +public class ModuleAttributeConstants { + + + final public static String A_DNS_SERVER_IP_LABEL = "dns-server-ip"; + + /** + * Fetch truncation codes present in {@link CrawlURI} annotations. + * All truncation annotations have a TRUNC_SUFFIX suffix (TODO: + * Make for-sure unique or redo truncation so definitive flag marked + * against {@link CrawlURI}). + */ + public static final String TRUNC_SUFFIX = "Trunc"; + // headerTrunc + public static final String HEADER_TRUNC = "header" + TRUNC_SUFFIX; + // timeTrunc + public static final String TIMER_TRUNC = "time" + TRUNC_SUFFIX; + // lenTrunc + public static final String LENGTH_TRUNC = "len" + TRUNC_SUFFIX; + + /** a 'source' (usu. URI) that's inherited by discovered URIs */ + public static final String A_SOURCE_TAG = "source"; + + public static final String A_HTTP_TRANSACTION = "http-transaction"; + + public static final String A_FETCH_BEGAN_TIME= "fetch-began-time"; + + private ModuleAttributeConstants() { + } + +} diff --git a/modules/src/main/java/org/archive/modules/PostProcessor.java b/modules/src/main/java/org/archive/modules/PostProcessor.java new file mode 100644 index 00000000..bc9fe3c5 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/PostProcessor.java @@ -0,0 +1,14 @@ +/** + * + */ +package org.archive.modules; + +/** + * Tagging interface for post processors. Needed so that ToeThreads can + * properly handle {@link ProcessResult#FINISH}. + * + * @author pjack + */ +public interface PostProcessor { + +} diff --git a/modules/src/main/java/org/archive/modules/ProcessResult.java b/modules/src/main/java/org/archive/modules/ProcessResult.java new file mode 100644 index 00000000..3de1929c --- /dev/null +++ b/modules/src/main/java/org/archive/modules/ProcessResult.java @@ -0,0 +1,67 @@ +/* Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * ProcessStatus.java + * Created on December 13, 2006 + * + * $Header$ + */ +package org.archive.modules; + +public class ProcessResult { + + + final public static ProcessResult PROCEED = + new ProcessResult(ProcessStatus.PROCEED); + + final public static ProcessResult FINISH = + new ProcessResult(ProcessStatus.FINISH); + + final public static ProcessResult STUCK = + new ProcessResult(ProcessStatus.STUCK); + + + final private ProcessStatus status; + final private String jumpTarget; + + + private ProcessResult(ProcessStatus status) { + this(status, null); + } + + + private ProcessResult(ProcessStatus status, String jumpName) { + this.status = status; + this.jumpTarget = jumpName; + } + + + public ProcessStatus getProcessStatus() { + return status; + } + + + public String getJumpTarget() { + return jumpTarget; + } + + + public static ProcessResult jump(String jumpTarget) { + return new ProcessResult(ProcessStatus.JUMP, jumpTarget); + } +} diff --git a/modules/src/main/java/org/archive/modules/ProcessStatus.java b/modules/src/main/java/org/archive/modules/ProcessStatus.java new file mode 100644 index 00000000..4b3e2c40 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/ProcessStatus.java @@ -0,0 +1,63 @@ +/* Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * ProcessStatus.java + * Created on December 13, 2006 + * + * $Header$ + */ +package org.archive.modules; + + +/** + * Returned by a Processor's process method to indicate the status of the + * process. + * + * @author pjack + */ +public enum ProcessStatus { + + /** + * The URI was processed normally, and no special action needs to be taken + * by the framework. + */ + PROCEED, + + /** + * The Processor believes that the ProcessorURI is invalid, or otherwise + * incapable of further processing at this time. The framework should not + * send the URI to any more processors, but should instead perform any + * necessary cleanup or post-processing on the URI. + */ + FINISH, + + /** + * The Processor has specified the next processor for the URI. The + * framework should send the URI to that processor instead of the reguarly + * scheduled next processor. + */ + JUMP, + + /** + * The Processor believes that futher processing of any ProcessorURIs is + * impossible at this point. For instance, if a Processor detects that + * a network interface is unavailable, or that a disk is full. + */ + STUCK + +} diff --git a/modules/src/main/java/org/archive/modules/Processor.java b/modules/src/main/java/org/archive/modules/Processor.java new file mode 100644 index 00000000..131d2f6c --- /dev/null +++ b/modules/src/main/java/org/archive/modules/Processor.java @@ -0,0 +1,261 @@ +/* Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Processor.java + * Created on October 5, 2006 + * + * $Header$ + */ +package org.archive.modules; + + +import java.io.Serializable; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.httpclient.HttpStatus; +import org.archive.modules.credential.CredentialAvatar; +import org.archive.modules.credential.Rfc2617Credential; +import org.archive.modules.deciderules.AcceptDecideRule; +import org.archive.modules.deciderules.DecideResult; +import org.archive.modules.deciderules.DecideRule; +import org.archive.net.UURI; +import org.archive.spring.HasKeyedProperties; +import org.archive.spring.KeyedProperties; +import org.springframework.context.Lifecycle; + + +/** + * A processor of URIs. The URI provides the context for the process; + * settings can be altered based on the URI. + * + * @author pjack + */ +public abstract class Processor +implements Serializable, HasKeyedProperties, Lifecycle { + protected KeyedProperties kp = new KeyedProperties(); + public KeyedProperties getKeyedProperties() { + return kp; + } + + String name = this.getClass().getSimpleName(); + public String getName() { + return this.name; + } + public void setName(String name) { + this.name = name; + } + + /** + * Whether or not this process will execute for a particular URI. + * If this is false for a URI, then the URI isn't processed, + * regardless of what the DecideRules say. + */ + { + setEnabled(true); + } + public boolean getEnabled() { + return (Boolean) kp.get("enabled"); + } + public void setEnabled(boolean enabled) { + kp.put("enabled",enabled); + } + + + /** + * Decide rule(s) (also particular to a URI) that determine whether + * or not a particular URI is processed here. If the rule(s) answer + * REJECT, processing is skipped. (ACCEPT or PASS allow processing + * to continue). + */ + { + setShouldProcessRule(new AcceptDecideRule()); + } + public DecideRule getShouldProcessRule() { + return (DecideRule) kp.get("shouldProcessRule"); + } + public void setShouldProcessRule(DecideRule rule) { + kp.put("shouldProcessRule", rule); + } + + /** + * The number of URIs processed by this processor. + */ + private AtomicLong uriCount = new AtomicLong(0); + + + /** + * Processes the given URI. First checks {@link #ENABLED} and + * {@link #DECIDE_RULES}. If ENABLED is false, then nothing happens. + * If the DECIDE_RULES indicate REJECT, then the + * {@link #innerRejectProcess(ProcessorURI)} method is invoked, and + * the process method returns. + * + *

      Next, the {@link #shouldProcess(ProcessorURI)} method is + * consulted to see if this Processor knows how to handle the given + * URI. If it returns false, then nothing futher occurs. + * + *

      FIXME: Should innerRejectProcess be called when ENABLED is false, + * or when shouldProcess returns false? The previous Processor + * implementation didn't handle it that way. + * + *

      Otherwise, the URI is considered valid. This processor's count + * of handled URIs is incremented, and the + * {@link #innerProcess(ProcessorURI)} method is invoked to actually + * perform the process. + * + * @param uri The URI to process + * @throws InterruptedException if the thread is interrupted + */ + public ProcessResult process(ProcessorURI uri) + throws InterruptedException { + if (!getEnabled()) { + return ProcessResult.PROCEED; + } + + if (getShouldProcessRule().decisionFor(uri) == DecideResult.REJECT) { + innerRejectProcess(uri); + return ProcessResult.PROCEED; + } + + if (shouldProcess(uri)) { + uriCount.incrementAndGet(); + return innerProcessResult(uri); + } else { + return ProcessResult.PROCEED; + } + } + + /** + * Returns the number of URIs this processor has handled. The returned + * number does not include URIs that were rejected by the + * {@link #ENABLED} flag, by the {@link #DECIDE_RULES}, or by the + * {@link #shouldProcess(ProcessorURI)} method. + * + * @return the number of URIs this processor has handled + */ + public long getURICount() { + return uriCount.get(); + } + + + /** + * Determines whether the given uri should be processed by this + * processor. For instance, a processor that only works on HTML + * content might reject the URI if its content type is not + * "text/html", if its content length is zero, and so on. + * + * @param uri the URI to test + * @return true if this processor should process that uri; false if not + */ + protected abstract boolean shouldProcess(ProcessorURI uri); + + + protected ProcessResult innerProcessResult(ProcessorURI uri) + throws InterruptedException { + innerProcess(uri); + return ProcessResult.PROCEED; + } + + /** + * Actually performs the process. By the time this method is invoked, + * it is known that the given URI passes the {@link #ENABLED}, the + * {@link #DECIDE_RULES} and the {@link #shouldProcess(ProcessorURI)} + * tests. + * + * @param uri the URI to process + * @throws InterruptedException if the thread is interrupted + */ + protected abstract void innerProcess(ProcessorURI uri) + throws InterruptedException; + + + /** + * Invoked after a URI has been rejected. The default implementation + * does nothing; subclasses may override to log rejects or something. + * + * @param uri the URI that was rejected + * @throws InterruptedException if the thread is interrupted + */ + protected void innerRejectProcess(ProcessorURI uri) + throws InterruptedException { + } + + + public static String flattenVia(ProcessorURI puri) { + UURI uuri = puri.getVia(); + return (uuri == null) ? "" : uuri.toString(); + } + + + public static boolean isSuccess(ProcessorURI puri) { + boolean result = false; + int statusCode = puri.getFetchStatus(); + if (statusCode == HttpStatus.SC_UNAUTHORIZED && + hasRfc2617CredentialAvatar(puri)) { + result = false; + } else { + result = (statusCode > 0); + } + return result; + } + + + public static long getRecordedSize(ProcessorURI puri) { + if (puri.getRecorder() == null) { + return puri.getContentSize(); + } else { + return puri.getRecorder().getRecordedInput().getSize(); + } + } + + + /** + * @return True if we have an rfc2617 payload. + */ + public static boolean hasRfc2617CredentialAvatar(ProcessorURI puri) { + Set avatars = puri.getCredentialAvatars(); + for (CredentialAvatar ca: avatars) { + if (ca.match(Rfc2617Credential.class)) { + return true; + } + } + return false; + } + + + // FIXME: Raise to interface + // FIXME: Internationalize somehow + // FIXME: Pass in PrintWriter instead creating large in-memory strings + public String report() { + return ""; + } + + boolean isRunning = false; + public boolean isRunning() { + return isRunning; + } + + public void start() { + isRunning = true; + } + + public void stop() { + isRunning = false; + } +} diff --git a/modules/src/main/java/org/archive/modules/ProcessorChain.java b/modules/src/main/java/org/archive/modules/ProcessorChain.java new file mode 100644 index 00000000..d1eb04dd --- /dev/null +++ b/modules/src/main/java/org/archive/modules/ProcessorChain.java @@ -0,0 +1,106 @@ +package org.archive.modules; + +import java.io.PrintWriter; +import java.util.Iterator; +import java.util.List; + +import org.archive.spring.HasKeyedProperties; +import org.archive.spring.KeyedProperties; +import org.archive.util.ArchiveUtils; +import org.archive.util.Reporter; +import org.springframework.context.Lifecycle; + +/** + * Collection of Processors to run. + * + * Not just a list on another bean so that: + * - chain is a prominent standalone part of configuration + * - Lifecycle events may be propagated to members defined + * as inner beans + * - future override capability may allow inserts at any place in + * order, not just end (assuming TBD specialized iterator) + * + */ +public class ProcessorChain +implements Iterable, + HasKeyedProperties, + Reporter, + Lifecycle { + + KeyedProperties kp = new KeyedProperties(); + public KeyedProperties getKeyedProperties() { + return kp; + } + + public int size() { + return getProcessors().size(); + } + + public Iterator iterator() { + return getProcessors().iterator(); + } + + @SuppressWarnings("unchecked") + public List getProcessors() { + return (List) kp.get("processors"); + } + public void setProcessors(List processors) { + kp.put("processors",processors); + } + + boolean isRunning = false; + public boolean isRunning() { + return isRunning; + } + + public void start() { + for(Processor p : getProcessors()) { + // relies on each Processor's start() being ok to call if + // already running, which is part of the Lifecycle contract + p.start(); + } + isRunning = true; + } + + public void stop() { + for(Processor p : getProcessors()) { + // relies on each Processor's stop() being ok to call if + // not running, which is part of the Lifecycle contract + p.stop(); + } + isRunning = false; + } + + /** + * Compiles and returns a human readable report on the active processors. + * @param writer Where to write to. + * @see org.archive.crawler.framework.Processor#report() + */ + public void reportTo(PrintWriter writer) { + writer.print( + "Processors report - " + + ArchiveUtils.get12DigitDate() + + "\n"); + + writer.print(" Number of Processors: " + size() + "\n"); + writer.print(" NOTE: Some processors may not return a report!\n\n"); + + for (Processor p: this) { + writer.print(p.report()); + } + } + + public String singleLineLegend() { + return ""; + } + + public void singleLineReportTo(PrintWriter pw) { + pw.print(size()); + pw.print(" processors: "); + for(Processor p : this) { + pw.print(p.getName()); + pw.print(" "); + } + } + +} diff --git a/modules/src/main/java/org/archive/modules/ProcessorLevel.java b/modules/src/main/java/org/archive/modules/ProcessorLevel.java new file mode 100644 index 00000000..6126eddc --- /dev/null +++ b/modules/src/main/java/org/archive/modules/ProcessorLevel.java @@ -0,0 +1,62 @@ +/* Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * ProcessorLevel.java + * Created on December 14, 2006 + * + * $Header$ + */ +package org.archive.modules; + +import java.util.logging.Level; + + +/** + * Custom log levels for processors. + * + * @author pjack + */ +public class ProcessorLevel extends Level { + + + private static final long serialVersionUID = 1L; + + + /** + * Level for logging problematic URIs. + */ + final public static ProcessorLevel URI = new ProcessorLevel("URI", + (WARNING.intValue() - INFO.intValue()) / 2 + INFO.intValue()); + // Halfway between INFO and WARNING + + private ProcessorLevel(String name, int level) { + super(name, level); + } + + + private Object readResolve() { + return URI; + } + + + public static void main(String args[]) { + System.out.println(Level.INFO.intValue()); + System.out.println(Level.WARNING.intValue()); + System.out.println(Level.SEVERE.intValue()); + } +} diff --git a/modules/src/main/java/org/archive/modules/ProcessorTestBase.java b/modules/src/main/java/org/archive/modules/ProcessorTestBase.java new file mode 100644 index 00000000..d26a6676 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/ProcessorTestBase.java @@ -0,0 +1,40 @@ +/* Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * ProcessorTest.java + * Created on October 5, 2006 + * + * $Header$ + */ +package org.archive.modules; + + + +import org.archive.state.ModuleTestBase; + + +/** + * Unit test for Processor. + * + * @author pjack + */ +public abstract class ProcessorTestBase extends ModuleTestBase { + +} + + diff --git a/modules/src/main/java/org/archive/modules/ProcessorURI.java b/modules/src/main/java/org/archive/modules/ProcessorURI.java new file mode 100644 index 00000000..b1d052db --- /dev/null +++ b/modules/src/main/java/org/archive/modules/ProcessorURI.java @@ -0,0 +1,143 @@ +/* Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * ProcessorURI.java + * Created on October 5, 2006 + * + * $Header$ + */ +package org.archive.modules; + + +import java.util.Collection; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.httpclient.HttpMethod; +import org.archive.modules.credential.CredentialAvatar; +import org.archive.modules.extractor.Link; +import org.archive.modules.extractor.LinkContext; +import org.archive.net.UURI; +import org.archive.util.Recorder; + + +/** + * A URI to be processed. + * + * @author pjack + */ +public interface ProcessorURI { + + + public static enum FetchType { HTTP_GET, HTTP_POST, UNKNOWN }; + + /** + * Returns the URI being processed. + * + * @return the URI + */ + UURI getUURI(); + + + UURI getVia(); + + boolean isPrerequisite(); + void setPrerequisite(boolean prereq); + + // Used to be a map attribute. + void setError(String msg); + + // Used to be a map attribute. + long getFetchBeginTime(); + void setFetchBeginTime(long time); + + // Used to be a map attribute + long getFetchCompletedTime(); + void setFetchCompletedTime(long time); + + String getDNSServerIPLabel(); + void setDNSServerIPLabel(String label); + + FetchType getFetchType(); + void setFetchType(FetchType type); + + Recorder getRecorder(); + + boolean containsDataKey(String attr); + Map getData(); + void makeHeritable(String attr); + Map getPersistentDataMap(); + void addPersistentDataMapKey(String s); + + String getUserAgent(); + void setUserAgent(String ua); + + long getContentSize(); + void setContentSize(long size); + + String getContentDigestSchemeString(); + byte[] getContentDigest(); + void setContentDigest(String algorithm, byte[] digest); + + String getContentType(); + void setContentType(String mimeType); + + long getContentLength(); + + Collection getAnnotations(); + Collection getNonFatalFailures(); + + int getFetchStatus(); + void setFetchStatus(int status); + + // Used to be a map attribute. May still want to be one. + HttpMethod getHttpMethod(); + void setHttpMethod(HttpMethod method); + + + boolean hasCredentialAvatars(); + Set getCredentialAvatars(); + + String getPathFromSeed(); + boolean isSeed(); + void setSeed(boolean seed); + + boolean isLocation(); + + + Collection getOutLinks(); + + UURI getBaseURI(); + void setBaseURI(UURI base); + + boolean hasBeenLinkExtracted(); + void linkExtractorFinished(); + + LinkContext getViaContext(); + + int getFetchAttempts(); + + boolean isSuccess(); + + void incrementDiscardedOutLinks(); + + String getSourceTag(); + + boolean forceFetch(); + +} diff --git a/modules/src/main/java/org/archive/modules/canonicalize/BaseRule.java b/modules/src/main/java/org/archive/modules/canonicalize/BaseRule.java new file mode 100644 index 00000000..fced885a --- /dev/null +++ b/modules/src/main/java/org/archive/modules/canonicalize/BaseRule.java @@ -0,0 +1,90 @@ +/* BaseRule + * + * Created on Oct 5, 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.modules.canonicalize; + +import java.io.Serializable; +import java.util.regex.Matcher; + +import org.archive.spring.HasKeyedProperties; +import org.archive.spring.KeyedProperties; + +/** + * Base of all rules applied canonicalizing a URL that are configurable + * via the Heritrix settings system. + * + * This base class is abstact. Subclasses must implement the + * {@link CanonicalizationRule#canonicalize(String, Object)} method. + * + * @author stack + * @version $Date$, $Revision$ + */ +public abstract class BaseRule +implements CanonicalizationRule, Serializable, HasKeyedProperties { + protected KeyedProperties kp = new KeyedProperties(); + public KeyedProperties getKeyedProperties() { + return kp; + } + + { + setEnabled(true); + } + public boolean getEnabled() { + return (Boolean) kp.get("enabled"); + } + public void setEnabled(boolean enabled) { + kp.put("enabled",enabled); + } + + /** + * Constructor. + */ + public BaseRule() { + } + + /** + * Run a regex that strips elements of a string. + * + * Assumes the regex has a form that wants to strip elements of the passed + * string. Assumes that if a match, appending group 1 + * and group 2 yields desired result. + * @param url Url to search in. + * @param matcher Matcher whose form yields a group 1 and group 2 if a + * match (non-null. + * @return Original url else concatenization of group 1 + * and group 2. + */ + protected String doStripRegexMatch(String url, Matcher matcher) { + return (matcher != null && matcher.matches())? + checkForNull(matcher.group(1)) + checkForNull(matcher.group(2)): + url; + } + + /** + * @param string String to check. + * @return string if non-null, else empty string (""). + */ + private String checkForNull(String string) { + return (string != null)? string: ""; + } + +} diff --git a/modules/src/main/java/org/archive/modules/canonicalize/CanonicalizationRule.java b/modules/src/main/java/org/archive/modules/canonicalize/CanonicalizationRule.java new file mode 100644 index 00000000..adf956dd --- /dev/null +++ b/modules/src/main/java/org/archive/modules/canonicalize/CanonicalizationRule.java @@ -0,0 +1,54 @@ +/* CanonicalizationRule + * + * Created on Oct 7, 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.modules.canonicalize; + +/** + * A rule to apply canonicalizing a url. + * @author stack + * @version $Date$, $Revision$ + */ +public interface CanonicalizationRule { + /** + * Apply this canonicalization rule. + * + * @param url Url string we apply this rule to. + * @param context An object that will provide context for the settings + * system. The UURI of the URL we're canonicalizing is an example of + * an object that provides context. + * @return Result of applying this rule to passed url. + */ + public String canonicalize(String url); + + /** + * @return Name of this rule. + */ +// public String getName(); + + /** + * @param context An object that will provide context for the settings + * system. The UURI of the URL we're canonicalizing is an example of + * an object that provides context. + * @return True if this rule is enabled and to be run. + */ + public boolean getEnabled(); +} diff --git a/modules/src/main/java/org/archive/modules/canonicalize/FixupQueryString.java b/modules/src/main/java/org/archive/modules/canonicalize/FixupQueryString.java new file mode 100644 index 00000000..377c4818 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/canonicalize/FixupQueryString.java @@ -0,0 +1,79 @@ +/* FixupQueryStr + * + * Created on Oct 5, 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.modules.canonicalize; + +/** + * Strip any trailing question mark. + * @author stack + * @version $Date$, $Revision$ + */ +public class FixupQueryString +extends BaseRule { + + private static final long serialVersionUID = 3L; + + /* + private static final String DESCRIPTION = + "Fixup the question mark that leads off the query string. " + + "This rule returns 'http://www.archive.org/index.html' if passed" + + " 'http://www.archive.org/index.html?'. It will also strip '?&'" + + " if '?&' is all that comprises the query string. Also strips" + + " extraneous leading '&': Returns 'http://archive.org/index.html?x=y" + + " if passed 'http://archive.org/index.html?&x=y." + + " Will also strip '&' if last thing in query string." + + " Operates on all schemes. This is a good rule to run toward the" + + " end of canonicalization processing."; + */ + + public FixupQueryString() { + } + + public String canonicalize(String url) { + if (url == null || url.length() <= 0) { + return url; + } + + int index = url.lastIndexOf('?'); + if (index > 0) { + if (index == (url.length() - 1)) { + // '?' is last char in url. Strip it. + url = url.substring(0, url.length() - 1); + } else if (url.charAt(index + 1) == '&') { + // Next char is '&'. Strip it. + if (url.length() == (index + 2)) { + // Then url ends with '?&'. Strip them. + url = url.substring(0, url.length() - 2); + } else { + // The '&' is redundant. Strip it. + url = url.substring(0, index + 1) + + url.substring(index + 2); + } + } else if (url.charAt(url.length() - 1) == '&') { + // If we have a lone '&' on end of query str, + // strip it. + url = url.substring(0, url.length() - 1); + } + } + return url; + } +} diff --git a/modules/src/main/java/org/archive/modules/canonicalize/LowercaseRule.java b/modules/src/main/java/org/archive/modules/canonicalize/LowercaseRule.java new file mode 100644 index 00000000..8595cbde --- /dev/null +++ b/modules/src/main/java/org/archive/modules/canonicalize/LowercaseRule.java @@ -0,0 +1,48 @@ +/* LowercaseRule + * + * Created on Oct 5, 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.modules.canonicalize; + +/** + * Lowercases the URL. + * @author stack + * @version $Date$, $Revision$ + */ +public class LowercaseRule +extends BaseRule { + + private static final long serialVersionUID = 3L; + +// private static final String DESCRIPTION = "Lowercase the URL. " + +// "Use this rule to lowercase the url. Operates on all schemes."; + + /** + * Constructor. + */ + public LowercaseRule() { + super(); + } + + public String canonicalize(String url) { + return url.toLowerCase(); + } +} diff --git a/modules/src/main/java/org/archive/modules/canonicalize/RegexRule.java b/modules/src/main/java/org/archive/modules/canonicalize/RegexRule.java new file mode 100644 index 00000000..a7d4af4c --- /dev/null +++ b/modules/src/main/java/org/archive/modules/canonicalize/RegexRule.java @@ -0,0 +1,90 @@ +/* RegexRule + * + * Created on Oct 6, 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.modules.canonicalize; + +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + + +/** + * General conversion rule. + * @author stack + * @version $Date$, $Revision$ + */ +public class RegexRule +extends BaseRule { + + private static final long serialVersionUID = -3L; + + protected static Logger logger = + Logger.getLogger(BaseRule.class.getName()); + +// private static final String DESCRIPTION = "General regex rule. " + +// "Specify a matching regex and a format string used outputting" + +// " result if a match was found. If problem compiling regex or" + +// " interpreting format, problem is logged, and this rule does" + +// " nothing. See User Manual for example usage."; + + + /** + * The regular expression to use to match. + */ + { + setRegex(Pattern.compile("(.*)")); + } + public Pattern getRegex() { + return (Pattern) kp.get("regex"); + } + public void setRegex(Pattern regex) { + kp.put("regex",regex); + } + + /** + * The format string to use when a match is found. + */ + { + setFormat("$1"); + } + public String getFormat() { + return (String) kp.get("format"); + } + public void setFormat(String format) { + kp.put("format",format); + } + + public RegexRule() { + } + + + public String canonicalize(String url) { + Pattern pattern = getRegex(); + Matcher matcher = pattern.matcher(url); + if (!matcher.matches()) { + return url; + } + StringBuffer buffer = new StringBuffer(url.length() * 2); + matcher.appendReplacement(buffer,getFormat()); + return buffer.toString(); + } +} \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/canonicalize/RulesCanonicalizationPolicy.java b/modules/src/main/java/org/archive/modules/canonicalize/RulesCanonicalizationPolicy.java new file mode 100644 index 00000000..205c8cdf --- /dev/null +++ b/modules/src/main/java/org/archive/modules/canonicalize/RulesCanonicalizationPolicy.java @@ -0,0 +1,99 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.modules.canonicalize; + +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.archive.spring.HasKeyedProperties; +import org.archive.spring.KeyedProperties; + +/** + * URI Canonicalizatioon Policy + * + * @contributor stack + * @contributor gojomo + */ +public class RulesCanonicalizationPolicy + extends UriCanonicalizationPolicy + implements HasKeyedProperties { + private static Logger logger = + Logger.getLogger(RulesCanonicalizationPolicy.class.getName()); + + protected KeyedProperties kp = new KeyedProperties(); + public KeyedProperties getKeyedProperties() { + return kp; + } + + { + setRules(getDefaultRules()); + } + @SuppressWarnings("unchecked") + public List getRules() { + return (List) kp.get("rules"); + } + public void setRules(List rules) { + kp.put("rules", rules); + } + + /** + * Run the passed uuri through the list of rules. + * @param context Url to canonicalize. + * @param rules Iterator of canonicalization rules to apply (Get one + * of these on the url-canonicalizer-rules element in order files or + * create a list externally). Rules must implement the Rule interface. + * @return Canonicalized URL. + */ + public String canonicalize(String before) { + String canonical = before; + if (logger.isLoggable(Level.FINER)) { + logger.finer("Canonicalizing: "+before); + } + for (CanonicalizationRule rule : getRules()) { + if(rule.getEnabled()) { + canonical = rule.canonicalize(canonical); + } + if (logger.isLoggable(Level.FINER)) { + logger.finer( + "Rule " + rule.getClass().getName() + " " + + (rule.getEnabled() + ? canonical :" (disabled)")); + } + } + return canonical; + } + + /** + * A reasonable set of default rules to use, if no others are + * provided by operator configuration. + */ + public static List getDefaultRules() { + List rules = new ArrayList(6); + rules.add(new LowercaseRule()); + rules.add(new StripUserinfoRule()); + rules.add(new StripWWWNRule()); + rules.add(new StripSessionIDs()); + rules.add(new StripSessionCFIDs()); + rules.add(new FixupQueryString()); + return rules; + } +} diff --git a/modules/src/main/java/org/archive/modules/canonicalize/StripExtraSlashes.java b/modules/src/main/java/org/archive/modules/canonicalize/StripExtraSlashes.java new file mode 100644 index 00000000..7ef8b5b8 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/canonicalize/StripExtraSlashes.java @@ -0,0 +1,48 @@ +/*RELICENSE-RESEARCH*/ +/* + * Created on 2006-aug-25 + * + * Copyright (C) 2006 Royal Library of Sweden. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package org.archive.modules.canonicalize; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Strip any extra slashes, '/', found in the path. + * Use this rule to equate 'http://www.archive.org//A//B/index.html' and + * 'http://www.archive.org/A/B/index.html'." + */ +public class StripExtraSlashes extends BaseRule { + private static final long serialVersionUID = 1L; + + private static final Pattern REGEX = Pattern.compile("(^https?://.*?)//+(.*)"); + + public StripExtraSlashes() { + super(); + } + + public String canonicalize(String url) { + Matcher matcher = REGEX.matcher(url); + while (matcher.matches()) { + url = matcher.group(1) + "/" + matcher.group(2); + matcher = REGEX.matcher(url); + } + return url; + } +} diff --git a/modules/src/main/java/org/archive/modules/canonicalize/StripSessionCFIDs.java b/modules/src/main/java/org/archive/modules/canonicalize/StripSessionCFIDs.java new file mode 100644 index 00000000..5d10a8cc --- /dev/null +++ b/modules/src/main/java/org/archive/modules/canonicalize/StripSessionCFIDs.java @@ -0,0 +1,68 @@ +/* $Id$ + * + * Created on September 1st, 2006 + * + * Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.modules.canonicalize; + +import java.util.regex.Pattern; + +/** + * Strip cold fusion session ids. + * @author stack + * @version $Date$, $Revision$ + */ +public class StripSessionCFIDs +extends BaseRule { + + private static final long serialVersionUID = 3L; + + private static final String REGEX = "^(.+)" + + "(?:cfid=[^&]+&cftoken=[^&]+(?:jsession=[^&]+)?)(?:&(.*))?$"; + +// private static final String DESCRIPTION = "Strip ColdFusion session IDs. " + +// "Use this rule to remove sessionids that look like the following: " + +// "CFID=12412453&CFTOKEN=15501799 or " + +// "CFID=3304324&CFTOKEN=57491900&jsessionid=a63098d96360$B0$D9$A " + +// "using the following case-insensitive regex: " + REGEX; + + /** + * Examples: + * 

      +     * Examples:
+     * boo?CFID=1169580&CFTOKEN=48630702&dtstamp=22%2F08%2F2006%7C06%3A58%3A11
+     * boo?CFID=12412453&CFTOKEN=15501799&dt=19_08_2006_22_39_28
+     * boo?CFID=14475712&CFTOKEN=2D89F5AF-3048-2957-DA4EE4B6B13661AB&r=468710288378&m=forgotten
+     * boo?CFID=16603925&CFTOKEN=2AE13EEE-3048-85B0-56CEDAAB0ACA44B8&r=501652357733&l1=home
+     * boo?CFID=3304324&CFTOKEN=57491900&jsessionid=a63098d96360$B0$D9$A 
+     *
      + */ + private static final Pattern COLDFUSION_PATTERN = + Pattern.compile(REGEX, Pattern.CASE_INSENSITIVE); + + + public StripSessionCFIDs() { + } + + public String canonicalize(String url) { + return doStripRegexMatch(url, COLDFUSION_PATTERN.matcher(url)); + } + +} \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/canonicalize/StripSessionIDs.java b/modules/src/main/java/org/archive/modules/canonicalize/StripSessionIDs.java new file mode 100644 index 00000000..921d13de --- /dev/null +++ b/modules/src/main/java/org/archive/modules/canonicalize/StripSessionIDs.java @@ -0,0 +1,82 @@ +/* StripSessionIDs + * + * Created on Oct 6, 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.modules.canonicalize; + +import java.util.regex.Pattern; + +/** + * Strip known session ids. + * @author stack + * @version $Date$, $Revision$ + */ +public class StripSessionIDs +extends BaseRule { + + private static final long serialVersionUID = 3L; + +// private static final String DESCRIPTION = "Strip known session IDs. " + +// "Use this rule to remove all of a set of known session IDs." + +// " For example, this rule will strip JSESSIONID and its value from" + +// " 'http://archive.org/index.html?" + +// "JSESSIONID=DDDSSE233232333355FFSXXXXDSDSDS'. The resulting" + +// " canonicalization returns 'http://archive.org/index.html'." + +// " This rule strips JSESSIONID, ASPSESSIONID, PHPSESSID, and 'sid'" + +// " session ids."; + + /** + * Example: jsessionid=999A9EF028317A82AC83F0FDFE59385A. + * Example: PHPSESSID=9682993c8daa2c5497996114facdc805. + */ + private static final Pattern BASE_PATTERN = Pattern.compile("^(.+)" + + "(?:(?:(?:jsessionid)|(?:phpsessid))=" + + "[0-9a-zA-Z]{32})(?:&(.*))?$", Pattern.CASE_INSENSITIVE); + + /** + * Example: sid=9682993c8daa2c5497996114facdc805. + * 'sid=' can be tricky but all sid= followed by 32 byte string + * so far seen have been session ids. Sid is a 32 byte string + * like the BASE_PATTERN only 'sid' is the tail of 'phpsessid' + * so have to have it run after the phpsessid elimination. + */ + private static final Pattern SID_PATTERN = + Pattern.compile("^(.+)" + + "(?:sid=[0-9a-zA-Z]{32})(?:&(.*))?$", Pattern.CASE_INSENSITIVE); + + /** + * Example:ASPSESSIONIDAQBSDSRT=EOHBLBDDPFCLHKPGGKLILNAM. + */ + private static final Pattern ASPSESSION_PATTERN = + Pattern.compile("^(.+)" + + "(?:ASPSESSIONID[a-zA-Z]{8}=[a-zA-Z]{24})(?:&(.*))?$", + Pattern.CASE_INSENSITIVE); + + public StripSessionIDs() { + } + + public String canonicalize(String url) { + url = doStripRegexMatch(url, BASE_PATTERN.matcher(url)); + url = doStripRegexMatch(url, SID_PATTERN.matcher(url)); + url = doStripRegexMatch(url, ASPSESSION_PATTERN.matcher(url)); + return url; + } +} \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/canonicalize/StripUserinfoRule.java b/modules/src/main/java/org/archive/modules/canonicalize/StripUserinfoRule.java new file mode 100644 index 00000000..4d114d39 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/canonicalize/StripUserinfoRule.java @@ -0,0 +1,55 @@ +/* StripUserinfoRule + * + * Created on Oct 5, 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.modules.canonicalize; + +import java.util.regex.Pattern; + +/** + * Strip any 'userinfo' found on http/https URLs. + * @author stack + * @version $Date$, $Revision$ + */ +public class StripUserinfoRule extends BaseRule { + + private static final long serialVersionUID = 3L; + +// private static final String DESCRIPTION = "Strip any 'userinfo' found. " + +// "Use this rule to equate 'http://stack:psswrd@archive.org/index.htm'" + +// " and 'http://archive.org/index.htm'. The resulting canonicalization" + +// " returns 'http://archive.org/index.htm'. Removes any userinfo" + +// " found. Operates on http/https/ftp/ftps schemes only."; + + /** + * Strip userinfo. + */ + private static final Pattern REGEX = + Pattern.compile("^((?:(?:https?)|(?:ftps?))://)(?:[^/]+@)(.*)$", + Pattern.CASE_INSENSITIVE); + + public StripUserinfoRule() { + } + + public String canonicalize(String url) { + return doStripRegexMatch(url, REGEX.matcher(url)); + } +} diff --git a/modules/src/main/java/org/archive/modules/canonicalize/StripWWWNRule.java b/modules/src/main/java/org/archive/modules/canonicalize/StripWWWNRule.java new file mode 100644 index 00000000..ac57aa2e --- /dev/null +++ b/modules/src/main/java/org/archive/modules/canonicalize/StripWWWNRule.java @@ -0,0 +1,62 @@ +/* StripWWWRule + * + * Created on Oct 5, 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.modules.canonicalize; + +import java.util.regex.Pattern; + +/** + * Strip any 'www[0-9]*' found on http/https URLs IF they have some + * path/query component (content after third slash). Top 'slash page' + * URIs are left unstripped: we prefer crawling redundant + * top pages to missing an entire site only available from either + * the www-full or www-less hostname, but not both. + * @author stack + * @version $Date$, $Revision$ + */ +public class StripWWWNRule extends BaseRule { + private static final long serialVersionUID = 3L; + +// private static final String DESCRIPTION = "Strip any 'www[0-9]*' found. " + +// "Use this rule to equate 'http://www.archive.org/index.html' and " + +// "'http://www0001.archive.org/index.html' with " + +// "'http://archive.org/index.html'. The resulting canonicalization " + +// "returns 'http://archive.org/index.html'. It removes any www's " + +// "or wwwNNN's found, where 'N' is one or more numerics, EXCEPT " + +// "on URIs that have no path/query component " + +// ". Top-level 'slash page' URIs are left unstripped: we prefer " + +// "crawling redundant top pages to missing an entire site only " + +// "available from either the www-full or www-less hostname, but not " + +// "both. Operates on http and https schemes only. " + +// "Use StripWWWRule to strip a lone 'www' only (This rule is a " + +// "more general version of StripWWWRule)."; + + private static final Pattern REGEX = + Pattern.compile("(?i)^(https?://)(?:www[0-9]*\\.)([^/]*/.+)$"); + + public StripWWWNRule() { + } + + public String canonicalize(String url) { + return doStripRegexMatch(url, REGEX.matcher(url)); + } +} \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/canonicalize/StripWWWRule.java b/modules/src/main/java/org/archive/modules/canonicalize/StripWWWRule.java new file mode 100644 index 00000000..fc7e7d37 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/canonicalize/StripWWWRule.java @@ -0,0 +1,58 @@ +/* StripWWWRule + * + * Created on Oct 5, 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.modules.canonicalize; + +import java.util.regex.Pattern; + +/** + * Strip any 'www' found on http/https URLs, IF they have some + * path/query component (content after third slash). (Top 'slash page' + * URIs are left unstripped, so that we prefer crawling redundant + * top pages to missing an entire site only available from either + * the www-full or www-less hostname, but not both). + * @author stack + * @version $Date$, $Revision$ + */ +public class StripWWWRule extends BaseRule { + + private static final long serialVersionUID = 3L; + +// private static final String DESCRIPTION = "Strip any 'www' found. " + +// "Use this rule to equate 'http://www.archive.org/index.html' and" + +// " 'http://archive.org/index.html'. The resulting canonicalization" + +// " returns 'http://archive.org/index.html'. It removes any www's " + +// "found, except on URIs that have no path/query component " + +// "('slash' pages). Operates on http and https schemes only. " + +// "Use the more general StripWWWNRule if you want to strip both 'www' " + +// "and 'www01', 'www02', etc."; + + private static final Pattern REGEX = + Pattern.compile("(?i)^(https?://)(?:www\\.)([^/]*/.+)$"); + + public StripWWWRule() { + } + + public String canonicalize(String url) { + return doStripRegexMatch(url, REGEX.matcher(url)); + } +} diff --git a/modules/src/main/java/org/archive/modules/canonicalize/UriCanonicalizationPolicy.java b/modules/src/main/java/org/archive/modules/canonicalize/UriCanonicalizationPolicy.java new file mode 100644 index 00000000..ec65f690 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/canonicalize/UriCanonicalizationPolicy.java @@ -0,0 +1,11 @@ +package org.archive.modules.canonicalize; + +/** + * URI Canonicalizatioon Policy + * + * @contributor stack + * @contributor gojomo + */ +public abstract class UriCanonicalizationPolicy { + public abstract String canonicalize(String uri); +} diff --git a/modules/src/main/java/org/archive/modules/credential/Credential.java b/modules/src/main/java/org/archive/modules/credential/Credential.java new file mode 100644 index 00000000..d612f78d --- /dev/null +++ b/modules/src/main/java/org/archive/modules/credential/Credential.java @@ -0,0 +1,241 @@ +/* Credential + * + * Created on Apr 1, 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.modules.credential; + +import java.io.Serializable; +import java.util.Iterator; +import java.util.Set; +import java.util.logging.Logger; + +import javax.management.AttributeNotFoundException; + +import org.apache.commons.httpclient.HttpClient; +import org.apache.commons.httpclient.HttpMethod; +import org.archive.modules.ProcessorURI; +import org.archive.modules.net.CrawlServer; +import org.archive.modules.net.ServerCache; +import org.archive.modules.net.ServerCacheUtil; + +/** + * Credential type. + * + * Let this be also a credential in the JAAS sense to in that this is what + * gets added to a subject on successful authentication since it contains + * data needed to authenticate (realm, login, password, etc.). + * + *

      Settings system assumes that subclasses implement a constructor that + * takes a name only. + * + * @author stack + * @version $Revision$, $Date$ + */ +public abstract class Credential implements Serializable { + + private static final Logger logger = + Logger.getLogger(Credential.class.getName()); + + + /** + * The root domain this credential goes against: E.g. www.archive.org + */ + String domain = ""; + /** + * @param context Context to use when searching for credential domain. + * @return The domain/root URI this credential is to go against. + * @throws AttributeNotFoundException If attribute not found. + */ + public String getDomain() { + return this.domain; + } + public void setDomain(String domain) { + this.domain = domain; + } + + /** + * Constructor. + */ + public Credential() { + } + + /** + * @param context Context to use when searching for credential domain. + * @param domain New domain. + * @throws AttributeNotFoundException + * @throws InvalidAttributeValueException + */ + /* + public void setCredentialDomain(CrawlerSettings context, String domain) + throws InvalidAttributeValueException, AttributeNotFoundException { + setAttribute(context, new Attribute(ATTR_CREDENTIAL_DOMAIN, domain)); + } + */ + + /** + * Attach this credentials avatar to the passed curi . + * + * Override if credential knows internally what it wants to attach as + * payload. Otherwise, if payload is external, use the below + * {@link #attach(ProcessorURI, String)}. + * + * @param curi ProcessorURI to load with credentials. + */ + public void attach(ProcessorURI curi) { + attach(curi, null); + } + + /** + * Attach this credentials avatar to the passed curi . + * + * @param curi ProcessorURI to load with credentials. + * @param payload Payload to carry in avatar. Usually credentials. + */ + public void attach(ProcessorURI curi, String payload) { + CredentialAvatar ca = (payload == null )? + new CredentialAvatar(this.getClass(), getKey()): + new CredentialAvatar(this.getClass(), getKey(), payload); + curi.getCredentialAvatars().add(ca); + } + + /** + * Detach this credential from passed curi. + * + * @param curi + * @return True if we detached a Credential reference. + */ + public boolean detach(ProcessorURI curi) { + boolean result = false; + Set avatars = curi.getCredentialAvatars(); + if (avatars.isEmpty()) { + logger.severe("This curi " + curi + " has no cred when it should"); + } + + Iterator iter = avatars.iterator(); + while (iter.hasNext()) { + CredentialAvatar ca = iter.next(); + if (ca.match(getClass(), getKey())) { + iter.remove(); + result = true; + } + } + + return result; + } + + /** + * Detach all credentials of this type from passed curi. + * + * @param curi + * @return True if we detached references. + */ + public boolean detachAll(ProcessorURI curi) { + boolean result = false; + Set avatars = curi.getCredentialAvatars(); + if (avatars.isEmpty()) { + logger.severe("This curi " + curi +" has no creds when it should."); + return false; + } + Iterator iter = avatars.iterator(); + while (iter.hasNext()) { + CredentialAvatar ca = iter.next(); + if (ca.match(getClass())) { + iter.remove(); + result = true; + } + } + return result; + } + + /** + * @param curi ProcessorURI to look at. + * @return True if this credential IS a prerequisite for passed + * ProcessorURI. + */ + public abstract boolean isPrerequisite(ProcessorURI curi); + + /** + * @param curi ProcessorURI to look at. + * @return True if this credential HAS a prerequisite for passed ProcessorURI. + */ + public abstract boolean hasPrerequisite(ProcessorURI curi); + + /** + * Return the authentication URI, either absolute or relative, that serves + * as prerequisite the passed curi. + * + * @param curi ProcessorURI to look at. + * @return Prerequisite URI for the passed curi. + */ + public abstract String getPrerequisite(ProcessorURI curi); + + /** + * @param context Context to use when searching for credential domain. + * @return Key that is unique to this credential type. + * @throws AttributeNotFoundException + */ + public abstract String getKey(); + + + /** + * @return True if this credential is of the type that needs to be offered + * on each visit to the server (e.g. Rfc2617 is such a type). + */ + public abstract boolean isEveryTime(); + + /** + * @param curi ProcessorURI to as for context. + * @param http Instance of httpclient. + * @param method Method to populate. + * @param payload Avatar payload to use populating the method. + * @return True if added a credentials. + */ + public abstract boolean populate(ProcessorURI curi, HttpClient http, + HttpMethod method, String payload); + + /** + * @param curi ProcessorURI to look at. + * @return True if this credential is to be posted. Return false if the + * credential is to be GET'd or if POST'd or GET'd are not pretinent to this + * credential type. + */ + public abstract boolean isPost(); + + /** + * Test passed curi matches this credentials rootUri. + * @param controller + * @param curi ProcessorURI to test. + * @return True if domain for credential matches that of the passed curi. + */ + public boolean rootUriMatch(ServerCache cache, + ProcessorURI curi) { + String cd = getDomain(); + + CrawlServer serv = ServerCacheUtil.getServerFor(cache, curi.getUURI()); + String serverName = serv.getName(); +// String serverName = controller.getServerCache().getServerFor(curi). +// getName(); + logger.fine("RootURI: Comparing " + serverName + " " + cd); + return cd != null && serverName != null && + serverName.equalsIgnoreCase(cd); + } + +} diff --git a/modules/src/main/java/org/archive/modules/credential/CredentialAvatar.java b/modules/src/main/java/org/archive/modules/credential/CredentialAvatar.java new file mode 100644 index 00000000..6dcd5e56 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/credential/CredentialAvatar.java @@ -0,0 +1,220 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.modules.credential; + +import java.io.Serializable; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.logging.Logger; + +import org.archive.modules.ProcessorURI; + +/** + * A credential representation. + * + * Added to the CrawlServer upon successful authentication. Used as a marker + * of successful authentication event and for carrying credential + * payload to be used subsequently doing preemptive authentications (e.g. + * For case of RFC2617, needs to be offered everytime we're accessing inside + * a protected area). Also carried by the ProcessorURI when cycling through + * processing chain trying a credential to see if it will authenticate. + * + *

      This class exists because its not safe to keep references + * to the settings derived Credential classes so instead of keeping references + * to credential classes, we carry around this avatar. + * + *

      Scope for avatars is crawlserver. Only used within a CrawlServer + * scope. + * + *

      Immutable. + * + * @author stack + * @version $Revision$, $Date$ + */ +public class CredentialAvatar +implements Serializable { + + private static final long serialVersionUID = 3L; + + private static final Logger logger = + Logger.getLogger(CredentialAvatar.class.getName()); + + /** + * Key for this credential avatar. + */ + private final String key; + + /** + * Type represented by this avatar. + */ + private final Class type; + + /** + * Data. + * + * May be null. + * + *

      This used to be an Object and I used to store in here + * the httpclient AuthScheme but AuthScheme is not serializable + * and so there'd be trouble getting this payload to lie down + * in a bdb database. Changed it to String. That should be + * generic enough for credential purposes. + */ + private final String payload; + + + /** + * Constructor. + * @param type Type for this credential avatar. + * @param key Key for this credential avatar. + */ + public CredentialAvatar(Class type, String key) { + this(type, key, null); + } + + /** + * Constructor. + * @param type Type for this credential avatar. + * @param key Key for this credential avatar. + * @param payload Data credential needs rerunning or preempting. May be + * null and then just the presence is used as signifier of successful + * auth. + */ + public CredentialAvatar(Class type, String key, String payload) { + if (!checkType(type)) { + throw new IllegalArgumentException("Type is unrecognized: " + + type); + } + this.key = key; + this.type = type; + this.payload = payload; + } + + /** + * Shutdown default constructor. + */ + @SuppressWarnings("unused") + private CredentialAvatar() { + super(); + this.key = null; + this.type = null; + this.payload = null; + } + + /** + * @param candidateType Type to check. + * @return True if this is a known credential type. + */ + protected boolean checkType(Class candidateType) { + boolean result = false; + List> types = CredentialStore.getCredentialTypes(); + for (Iterator> i = types.iterator(); i.hasNext();) { + if (i.next().equals(candidateType)) { + result = true; + break; + } + } + return result; + } + + /** + * @return Returns the payload. May be null. + */ + public String getPayload() { + return this.payload; + } + + /** + * @return Returns the key. + */ + public String getKey() { + return this.key; + } + + /** + * @return Type represented by this avatar. + */ + public Class getType() { + return this.type; + } + + /** + * @param otherType Class to match. + * @return True if this credential avatar is of same type. + */ + public boolean match(Class otherType) { + return this.type.equals(otherType); + } + + /** + * @param otherType Credential to match. + * @param otherKey Key to test. + * @return True if this is avatar for passed credential. + */ + public boolean match(Class otherType, String otherKey) { + return match(otherType) && + (otherKey != null && this.key != null && + this.key.equals(otherKey)); + } + + public String toString() { + return getType() + "." + this.getKey(); + } + + /** + * @param handler Settings handler. + * @param curi ProcessorURI to use for context. + * @return The credential this avatar represents. + */ + public Credential getCredential(CredentialStore cs, ProcessorURI curi) { + Credential result = null; + + if (cs == null) { + logger.severe("No credential store for " + curi); + return result; + } + + Collection all = cs.getAll(); + if (all == null) { + logger.severe("Have CredentialAvatar " + toString() + + " but no collection: " + curi); + return result; + } + + for (Credential c: all) { + if (!this.type.isInstance(c)) { + continue; + } + String credKey = c.getKey(); + if (credKey != null && credKey.equals(getKey())) { + result = c; + break; + } + } + + if (result == null) { + logger.severe("Have CredentialAvatar " + toString() + + " but no corresponding credential: " + curi); + } + + return result; + } +} diff --git a/modules/src/main/java/org/archive/modules/credential/CredentialStore.java b/modules/src/main/java/org/archive/modules/credential/CredentialStore.java new file mode 100644 index 00000000..2db2c329 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/credential/CredentialStore.java @@ -0,0 +1,213 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.modules.credential; + +import java.io.Serializable; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.logging.Logger; + +import javax.management.AttributeNotFoundException; +import javax.management.MBeanException; +import javax.management.ReflectionException; + +import org.archive.modules.ProcessorURI; +import org.archive.spring.HasKeyedProperties; +import org.archive.spring.KeyedProperties; + +/** + * Front door to the credential store. + * + * Come here to get at credentials. + * + *

      See Credential + * Store Design. + * + * @author stack + * @version $Revision$, $Date$ + */ +public class CredentialStore implements Serializable, HasKeyedProperties { + + private static final long serialVersionUID = 3L; + + private static Logger logger = Logger.getLogger( + "org.archive.crawler.datamodel.CredentialStore"); + + KeyedProperties kp = new KeyedProperties(); + public KeyedProperties getKeyedProperties() { + return kp; + } + + /** + * Credentials used by heritrix authenticating. See + * http://crawler.archive.org/proposals/auth/ for background. + * + * @see http://crawler.archive.org/proposals/auth/ + */ + { + setCredentials(new HashMap()); + } + @SuppressWarnings("unchecked") + public Map getCredentials() { + return (Map) kp.get("credentials"); + } + public void setCredentials(Map map) { + kp.put("credentials",map); + } + + /** + * List of possible credential types as a List. + * + * This types are inner classes of this credential type so they cannot + * be created without their being associated with a credential list. + */ + private static final List> credentialTypes; + // Initialize the credentialType data member. + static { + // Array of all known credential types. + Class [] tmp = {HtmlFormCredential.class, Rfc2617Credential.class}; + credentialTypes = Collections.unmodifiableList(Arrays.asList(tmp)); + } + + /** + * Constructor. + */ + public CredentialStore() { + } + + /** + * @return Unmodifable list of credential types. + */ + public static List> getCredentialTypes() { + return CredentialStore.credentialTypes; + } + + + /** + * @param context Pass a ProcessorURI. Used to set + * context. + * @return An iterator or null. + */ + public Collection getAll() { + Map map = getCredentials(); + return map.values(); + } + + /** + * @param context Used to set context. + * @param name Name to give the manufactured credential. Should be unique + * else the add of the credential to the list of credentials will fail. + * @return Returns name'd credential. + * @throws AttributeNotFoundException + * @throws MBeanException + * @throws ReflectionException + */ + public Credential get(/*StateProvider*/Object context, String name) { + return getCredentials().get(name); + } + + + /** + * Return set made up of all credentials of the passed + * type. + * + * @param context Used to set context. + * @param type Type of the list to return. Type is some superclass of + * credentials. + * @return Unmodifable sublist of all elements of passed type. + */ + public Set subset(ProcessorURI context, Class type) { + return subset(context, type, null); + } + + /** + * Return set made up of all credentials of the passed + * type. + * + * @param context Used to set context. + * @param type Type of the list to return. Type is some superclass of + * credentials. + * @param rootUri RootUri to match. May be null. In this case we return + * all. Currently we expect the CrawlServer name to equate to root Uri. + * Its not. Currently it doesn't distingush between servers of same name + * but different ports (e.g. http and https). + * @return Unmodifable sublist of all elements of passed type. + */ + public Set subset(ProcessorURI context, Class type, String rootUri) { + Set result = null; + for (Credential c: getAll()) { + if (!type.isInstance(c)) { + continue; + } + if (rootUri != null) { + String cd = c.getDomain(); + if (cd == null) { + continue; + } + if (!rootUri.equalsIgnoreCase(cd)) { + continue; + } + } + if (result == null) { + result = new HashSet(); + } + result.add(c); + } + return result; + } + + + public Credential getCredential(ProcessorURI curi, CredentialAvatar ca) { + Credential result = null; + + Collection all = getAll(); + if (all == null) { + logger.severe("Have CredentialAvatar " + toString() + + " but no collection: " + curi); + return result; + } + + for (Credential c: all) { + if (!ca.getType().isInstance(c)) { + continue; + } + String credKey = c.getKey(); + if (credKey != null && credKey.equals(ca.getKey())) { + result = c; + break; + } + } + + if (result == null) { + logger.severe("Have CredentialAvatar " + toString() + + " but no corresponding credential: " + curi); + } + + return result; + + } +} diff --git a/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java b/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java new file mode 100644 index 00000000..13b9183a --- /dev/null +++ b/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java @@ -0,0 +1,182 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.modules.credential; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.logging.Logger; + +import org.apache.commons.httpclient.HttpClient; +import org.apache.commons.httpclient.HttpMethod; +import org.apache.commons.httpclient.HttpMethodBase; +import org.apache.commons.httpclient.NameValuePair; +import org.apache.commons.httpclient.URIException; +import org.apache.commons.httpclient.methods.GetMethod; +import org.apache.commons.httpclient.methods.PostMethod; +import org.archive.modules.ProcessorURI; +import org.archive.net.UURI; +import org.archive.net.UURIFactory; + +/** + * Credential that holds all needed to do a GET/POST to a HTML form. + * + * @author stack + * @version $Revision$, $Date$ + */ +public class HtmlFormCredential extends Credential { + private static final long serialVersionUID = -3L; + + private static final Logger logger = + Logger.getLogger(HtmlFormCredential.class.getName()); + + /** + * Full URI of page that contains the HTML login form we're to apply these + * credentials too: E.g. http://www.archive.org + */ + String loginUri = ""; + public String getLoginUri() { + return this.loginUri; + } + public void setLoginUri(String loginUri) { + this.loginUri = loginUri; + } + + /** + * Form items. + */ + Map formItems = new HashMap(); + public Map getFormItems() { + return this.formItems; + } + public void setFormItems(Map formItems) { + this.formItems = formItems; + } + + + enum Method { + GET, + POST + } + /** + * GET or POST. + */ + Method httpMethod = Method.POST; + public Method getHttpMethod() { + return this.httpMethod; + } + public void setHttpMethod(Method method) { + this.httpMethod = method; + } + + /** + * Constructor. + */ + public HtmlFormCredential() { + } + + public boolean isPrerequisite(final ProcessorURI curi) { + boolean result = false; + String curiStr = curi.getUURI().toString(); + String loginUri = getPrerequisite(curi); + if (loginUri != null) { + try { + UURI uuri = UURIFactory.getInstance(curi.getUURI(), loginUri); + if (uuri != null && curiStr != null && + uuri.toString().equals(curiStr)) { + result = true; + if (!curi.isPrerequisite()) { + curi.setPrerequisite(true); + logger.fine(curi + " is prereq."); + } + } + } catch (URIException e) { + logger.severe("Failed to uuri: " + curi + ", " + + e.getMessage()); + } + } + return result; + } + + public boolean hasPrerequisite(ProcessorURI curi) { + return getPrerequisite(curi) != null; + } + + public String getPrerequisite(ProcessorURI curi) { + return getLoginUri(); + } + + public String getKey() { + return getLoginUri(); + } + + public boolean isEveryTime() { + // This authentication is one time only. + return false; + } + + public boolean populate(ProcessorURI curi, HttpClient http, HttpMethod method, + String payload) { + // http is not used. + // payload is not used. + boolean result = false; + Map formItems = getFormItems(); + if (formItems == null || formItems.size() <= 0) { + try { + logger.severe("No form items for " + method.getURI()); + } + catch (URIException e) { + logger.severe("No form items and exception getting uri: " + + e.getMessage()); + } + return result; + } + + NameValuePair[] data = new NameValuePair[formItems.size()]; + int index = 0; + String key = null; + for (Iterator i = formItems.keySet().iterator(); i.hasNext();) { + key = i.next(); + data[index++] = new NameValuePair(key, (String)formItems.get(key)); + } + if (method instanceof PostMethod) { + ((PostMethod)method).setRequestBody(data); + result = true; + } else if (method instanceof GetMethod) { + // Append these values to the query string. + // Get current query string, then add data, then get it again + // only this time its our data only... then append. + HttpMethodBase hmb = (HttpMethodBase)method; + String currentQuery = hmb.getQueryString(); + hmb.setQueryString(data); + String newQuery = hmb.getQueryString(); + hmb.setQueryString(((currentQuery != null)? currentQuery: "") + + "&" + newQuery); + result = true; + } else { + logger.severe("Unknown method type: " + method); + } + return result; + } + + public boolean isPost() { + return Method.POST.equals(getHttpMethod()); + } +} diff --git a/modules/src/main/java/org/archive/modules/credential/Rfc2617Credential.java b/modules/src/main/java/org/archive/modules/credential/Rfc2617Credential.java new file mode 100644 index 00000000..a4b56665 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/credential/Rfc2617Credential.java @@ -0,0 +1,179 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.modules.credential; + +import java.util.Iterator; +import java.util.Set; +import java.util.logging.Logger; + +import org.apache.commons.httpclient.HttpClient; +import org.apache.commons.httpclient.HttpMethod; +import org.apache.commons.httpclient.URIException; +import org.apache.commons.httpclient.UsernamePasswordCredentials; +import org.apache.commons.httpclient.auth.AuthScope; +import org.archive.modules.ProcessorURI; + +/** + * A Basic/Digest auth RFC2617 credential. + * + * @author stack + * @version $Revision$, $Date$ + */ +public class Rfc2617Credential extends Credential { + private static final long serialVersionUID = 3L; + + private static Logger logger = + Logger.getLogger(Rfc2617Credential.class.getName()); + + + /** Basic/Digest Auth realm. */ + String realm = "Realm"; + public String getRealm() { + return this.realm; + } + public void setRealm(String realm) { + this.realm = realm; + } + + /** Login. */ + String login = "login"; + public String getLogin() { + return this.login; + } + public void setLogin(String login) { + this.login = login; + } + + /** Password. */ + String password = "password"; + public String getPassword() { + return this.password; + } + public void setPassword(String password) { + this.password = password; + } + + /** + * Constructor. + */ + public Rfc2617Credential() { + } + + public boolean isPrerequisite(ProcessorURI curi) { + // Return false. Later when we implement preemptive + // rfc2617, this will change. + return false; + } + + public boolean hasPrerequisite(ProcessorURI curi) { + // Return false. Later when we implement preemptive + // rfc2617, this will change. + return false; + } + + public String getPrerequisite(ProcessorURI curi) { + // Return null. Later when we implement preemptive + // rfc2617, this will change. + return null; + } + + public String getKey() { + return getRealm(); + } + + public boolean isEveryTime() { + return true; + } + + public boolean populate(ProcessorURI curi, HttpClient http, HttpMethod method, + String payload) { + boolean result = false; + String authRealm = payload; + if (authRealm == null) { + logger.severe("No authscheme though creds: " + curi); + return result; + } + + // Always add the credential to HttpState. Doing this because no way of + // removing the credential once added AND there is a bug in the + // credentials management system in that it always sets URI root to + // null: it means the key used to find a credential is NOT realm + root + // URI but just the realm. Unless I set it everytime, there is + // possibility that as this thread progresses, it might come across a + // realm already loaded but the login and password are from another + // server. We'll get a failed authentication that'd be difficult to + // explain. + // + // Have to make a UsernamePasswordCredentials. The httpclient auth code + // does an instanceof down in its guts. + UsernamePasswordCredentials upc = null; + try { + upc = new UsernamePasswordCredentials(getLogin(), + getPassword()); + http.getState().setCredentials(new AuthScope(curi.getUURI().getHost(), + curi.getUURI().getPort(), authRealm), upc); + logger.fine("Credentials for realm " + authRealm + + " for CrawlURI " + curi.toString() + " added to request: " + + result); + result = true; + } catch (URIException e) { + logger.severe("Failed to parse host from " + curi + ": " + + e.getMessage()); + } + + return result; + } + + public boolean isPost() { + // Return false. This credential type doesn't care whether posted or + // get'd. + return false; + } + + /** + * Convenience method that does look up on passed set using realm for key. + * + * @param rfc2617Credentials Set of Rfc2617 credentials. If passed set is + * not pure Rfc2617Credentials then will be ClassCastExceptions. + * @param realm Realm to find in passed set. + * @param context Context to use when searching the realm. + * @return Credential of passed realm name else null. If more than one + * credential w/ passed realm name, and there shouldn't be, we return first + * found. + */ + public static Rfc2617Credential getByRealm(Set rfc2617Credentials, + String realm, ProcessorURI context) { + + Rfc2617Credential result = null; + if (rfc2617Credentials == null || rfc2617Credentials.size() <= 0) { + return result; + } + if (rfc2617Credentials != null && rfc2617Credentials.size() > 0) { + for (Iterator i = rfc2617Credentials.iterator(); i.hasNext();) { + Rfc2617Credential c = (Rfc2617Credential)i.next(); + if (c.getRealm().equals(realm)) { + result = c; + break; + } + } + } + return result; + } +} diff --git a/modules/src/main/java/org/archive/modules/credential/package.html b/modules/src/main/java/org/archive/modules/credential/package.html new file mode 100644 index 00000000..baf9acb6 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/credential/package.html @@ -0,0 +1,16 @@ + + + +org.archive.io.arc package + + +Contains html form login and basic and digest credentials +used by Heritrix logging into sites. + +

      To watch credentials running, enable logging setting the following +logging level for FetchHttp class: +org.archive.crawler.fetcher.FetchHTTP.level = FINE

      + + + + diff --git a/modules/src/main/java/org/archive/modules/deciderules/AcceptDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/AcceptDecideRule.java new file mode 100644 index 00000000..06ab7f9d --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/AcceptDecideRule.java @@ -0,0 +1,44 @@ +/* Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * AcceptDecideRule.java + * Created on October 5, 2006 + * + * $Header$ + */ +package org.archive.modules.deciderules; + +import org.archive.modules.ProcessorURI; + + +public class AcceptDecideRule extends DecideRule { + private static final long serialVersionUID = 3L; + + + @Override + protected DecideResult innerDecide(ProcessorURI uri) { + return DecideResult.ACCEPT; + } + + + @Override + public DecideResult onlyDecision(ProcessorURI uri) { + return DecideResult.ACCEPT; + } + +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/AddRedirectFromRootServerToScope.java b/modules/src/main/java/org/archive/modules/deciderules/AddRedirectFromRootServerToScope.java new file mode 100644 index 00000000..19595007 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/AddRedirectFromRootServerToScope.java @@ -0,0 +1,73 @@ +/* AddRedirectFromRootServerToScope + * + * Created on May 25, 2005 + * + * Copyright (C) 2005 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.modules.deciderules; + +import java.util.logging.Logger; +import org.apache.commons.httpclient.URIException; +import org.archive.modules.ProcessorURI; +import org.archive.net.UURI; + + +public class AddRedirectFromRootServerToScope +extends PredicatedDecideRule { + + private static final long serialVersionUID = 3L; + + private static final Logger LOGGER = + Logger.getLogger(AddRedirectFromRootServerToScope.class.getName()); + private static final String SLASH = "/"; + + public AddRedirectFromRootServerToScope() { + } + + @Override + protected boolean evaluate(ProcessorURI uri) { + UURI via = uri.getVia(); + if (via == null) { + return false; + } + try { + String chost = uri.getUURI().getHostBasename(); + if (chost == null) { + return false; + } + + String viaHost = via.getHostBasename(); + if (viaHost == null) { + return false; + } + + if (chost.equals(viaHost) && uri.isLocation() + && via.getPath().equals(SLASH)) { + uri.setSeed(true); + LOGGER.info("Adding " + uri + " to seeds via " + via); + return true; + } + } catch (URIException e) { + e.printStackTrace(); + } + return false; + } + + +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/BeanShellDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/BeanShellDecideRule.java new file mode 100644 index 00000000..e8b134a7 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/BeanShellDecideRule.java @@ -0,0 +1,173 @@ +/* BeanShellDecideRule +* +* $Id$ +* +* Created on Aug 7, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.modules.deciderules; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.archive.modules.ProcessorURI; +import org.archive.spring.ConfigPath; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.annotation.Required; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; + +import bsh.EvalError; +import bsh.Interpreter; + + +/** + * Rule which runs a groovy script to make its decision. + * + * Script source may be provided via a file local to the crawler. + * + * Variables available to the script include 'object' (the object to be + * evaluated, typically a CandidateURI or CrawlURI), 'self' + * (this GroovyDecideRule instance), and 'controller' (the crawl's + * CrawlController instance). + * + * TODO: reduce copy & paste with GroovyProcessor + * + * @author gojomo + */ +public class BeanShellDecideRule extends DecideRule +implements ApplicationContextAware { + + private static final long serialVersionUID = 3L; + + private static final Logger logger = + Logger.getLogger(BeanShellDecideRule.class.getName()); + + /** BeanShell script file. */ + protected ConfigPath scriptFile = null; + public ConfigPath getScriptFile() { + return scriptFile; + } + @Required + public void setScriptFile(ConfigPath scriptFile) { + this.scriptFile = scriptFile; + } + + /** + * Whether each ToeThread should get its own independent script context, or + * they should share synchronized access to one context. Default is true, + * meaning each threads gets its own isolated context. + */ + protected boolean isolateThreads = true; + public boolean getIsolateThreads() { + return isolateThreads; + } + public void setIsolateThreads(boolean isolateThreads) { + this.isolateThreads = isolateThreads; + } + + ApplicationContext appCtx; + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + this.appCtx = applicationContext; + } + + protected ThreadLocal threadInterpreter = + new ThreadLocal();; + protected Interpreter sharedInterpreter; + public Map sharedMap = + Collections.synchronizedMap(new HashMap()); + protected boolean initialized = false; + + public BeanShellDecideRule() { + } + + @Override + public synchronized DecideResult innerDecide(ProcessorURI uri) { + // depending on previous configuration, interpreter may + // be local to this thread or shared + Interpreter interpreter = getInterpreter(); + synchronized(interpreter) { + // synchronization is harmless for local thread interpreter, + // necessary for shared interpreter + try { + interpreter.set("object",uri); + return (DecideResult)interpreter.eval("decisionFor(object)"); + } catch (EvalError e) { + // TODO Auto-generated catch block + e.printStackTrace(); + return DecideResult.PASS; + } + } + } + + /** + * Get the proper Interpreter instance -- either shared or local + * to this thread. + * @return Interpreter to use + */ + protected Interpreter getInterpreter() { + if(sharedInterpreter==null + && getIsolateThreads()) { + // initialize + sharedInterpreter = newInterpreter(); + } + if(sharedInterpreter!=null) { + return sharedInterpreter; + } + Interpreter interpreter = threadInterpreter.get(); + if(interpreter==null) { + interpreter = newInterpreter(); + threadInterpreter.set(interpreter); + } + return interpreter; + } + + /** + * Create a new Interpreter instance, preloaded with any supplied + * source file and the variables 'self' (this + * BeanShellProcessor) and 'controller' (the CrawlController). + * + * @return the new Interpreter instance + */ + protected Interpreter newInterpreter() { + Interpreter interpreter = new Interpreter(); + try { + interpreter.set("self", this); + interpreter.set("context", appCtx); + + File file = getScriptFile().getFile(); + try { + interpreter.source(file.getPath()); + } catch (IOException e) { + logger.log(Level.SEVERE,"unable to read script file",e); + } + } catch (EvalError e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + return interpreter; + } +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/ContentLengthDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/ContentLengthDecideRule.java new file mode 100644 index 00000000..e26340e8 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/ContentLengthDecideRule.java @@ -0,0 +1,61 @@ +/* $Id$ + * + * Created on 28.8.2006 + * + * Copyright (C) 2006 Olaf Freyer + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.modules.deciderules; + +import org.archive.modules.ProcessorURI; + +public class ContentLengthDecideRule extends DecideRule { + + private static final long serialVersionUID = 3L; + + + /** + * Content-length threshold. The rule returns ACCEPT if the content-length + * is less than this threshold, or REJECT otherwise. The default is + * 2^63, meaning any document will be accepted. + */ + { + setContentLengthThreshold(Long.MAX_VALUE); + } + public long getContentLengthThreshold() { + return (Long) kp.get("contentLengthThreshold"); + } + public void setContentLengthThreshold(long threshold) { + kp.put("contentLengthThreshold",threshold); + } + + /** + * Usual constructor. + */ + public ContentLengthDecideRule() { + } + + + protected DecideResult innerDecide(ProcessorURI uri) { + if (uri.getContentLength() < getContentLengthThreshold()) { + return DecideResult.ACCEPT; + } + return DecideResult.REJECT; + } + +} \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/deciderules/ContentTypeMatchesRegExpDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/ContentTypeMatchesRegExpDecideRule.java new file mode 100644 index 00000000..48d8da36 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/ContentTypeMatchesRegExpDecideRule.java @@ -0,0 +1,43 @@ +/* $Id: $ + * + * Copyright (C) 2007 Olaf Freyer + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.modules.deciderules; + +import org.archive.modules.ProcessorURI; + + +/** + * DecideRule whose decision is applied if the URI's content-type + * is present and matches the supplied regular expression. + * + * @author Olaf Freyer + */ +public class ContentTypeMatchesRegExpDecideRule extends MatchesRegExpDecideRule{ + private static final long serialVersionUID = -2066930281015155843L; + + public ContentTypeMatchesRegExpDecideRule() { + } + + + @Override + protected String getString(ProcessorURI uri) { + return uri.getContentType(); + } +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/ContentTypeNotMatchesRegExpDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/ContentTypeNotMatchesRegExpDecideRule.java new file mode 100644 index 00000000..20997ddc --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/ContentTypeNotMatchesRegExpDecideRule.java @@ -0,0 +1,51 @@ + +/* $Id: $ + * + * Copyright (C) 2007 Olaf Freyer + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.modules.deciderules; + +import org.archive.modules.ProcessorURI; + +/** + * DecideRule whose decision is applied if the URI's content-type + * is present and does not match the supplied regular expression. + * + * @author Olaf Freyer + */ +public class ContentTypeNotMatchesRegExpDecideRule extends + ContentTypeMatchesRegExpDecideRule { + private static final long serialVersionUID = 4729800377757426137L; + + public ContentTypeNotMatchesRegExpDecideRule() { + } + + /** + * Evaluate whether given object's string version does not match + * configured regexp (by reversing the superclass's answer). + * + * @param object Object to make decision about. + * @return true if the regexp is not matched + */ + @Override + protected boolean evaluate(ProcessorURI o) { + return !super.evaluate(o); + } + +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/DecideResult.java b/modules/src/main/java/org/archive/modules/deciderules/DecideResult.java new file mode 100644 index 00000000..ebaf5dd8 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/DecideResult.java @@ -0,0 +1,54 @@ +/* Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * DecideResult.java + * Created on October 5, 2006 + * + * $Header$ + */ +package org.archive.modules.deciderules; + + +/** + * The decision of a DecideRule. + * + * @author pjack + */ +public enum DecideResult { + + /** Indicates the URI was accepted. */ + ACCEPT, + + /** Indicates the URI was neither accepted nor rejected. */ + PASS, + + /** Indicates the URI was rejected. */ + REJECT; + + + public static DecideResult invert(DecideResult result) { + switch (result) { + case ACCEPT: + return REJECT; + case REJECT: + return ACCEPT; + default: + return result; + } + } +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/DecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/DecideRule.java new file mode 100644 index 00000000..e21ec972 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/DecideRule.java @@ -0,0 +1,85 @@ +/* Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * DecideRule.java + * Created on October 5, 2006 + * + * $Header$ + */ +package org.archive.modules.deciderules; + + +import java.io.Serializable; + +import org.archive.modules.ProcessorURI; +import org.archive.spring.HasKeyedProperties; +import org.archive.spring.KeyedProperties; + +public abstract class DecideRule implements Serializable, HasKeyedProperties { + protected KeyedProperties kp = new KeyedProperties(); + public KeyedProperties getKeyedProperties() { + return kp; + } + + { + setEnabled(true); + } + public boolean getEnabled() { + return (Boolean) kp.get("enabled"); + } + public void setEnabled(boolean enabled) { + kp.put("enabled",enabled); + } + + protected String comment = ""; + public String getComment() { + return comment; + } + public void setComment(String comment) { + this.comment = comment; + } + + public DecideRule() { + + } + + public DecideResult decisionFor(ProcessorURI uri) { + if (!getEnabled()) { + return DecideResult.PASS; + } + DecideResult result = innerDecide(uri); + if (result == DecideResult.PASS) { + return result; + } + + return result; + } + + + protected abstract DecideResult innerDecide(ProcessorURI uri); + + + public DecideResult onlyDecision(ProcessorURI uri) { + return null; + } + + public boolean accepts(ProcessorURI uri) { + return DecideResult.ACCEPT == decisionFor(uri); + } + +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/DecideRuleSequence.java b/modules/src/main/java/org/archive/modules/deciderules/DecideRuleSequence.java new file mode 100644 index 00000000..6c36de93 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/DecideRuleSequence.java @@ -0,0 +1,63 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.modules.deciderules; + +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.archive.modules.ProcessorURI; + +public class DecideRuleSequence extends DecideRule { + final private static Logger LOGGER = + Logger.getLogger(DecideRuleSequence.class.getName()); + private static final long serialVersionUID = 3L; + + @SuppressWarnings("unchecked") + public List getRules() { + return (List) kp.get("rules"); + } + public void setRules(List rules) { + kp.put("rules", rules); + } + + public DecideResult innerDecide(ProcessorURI uri) { + DecideResult result = DecideResult.PASS; + List rules = getRules(); + int max = rules.size(); + for (int i = 0; i < max; i++) { + DecideRule rule = rules.get(i); + if (rule.onlyDecision(uri) != result) { + DecideResult r = rule.decisionFor(uri); + if (LOGGER.isLoggable(Level.FINEST)) { + LOGGER.finest("DecideRule #" + i + " " + + rule.getClass().getName() + " returned " + r); + } + if (r != DecideResult.PASS) { + result = r; + } + } + } + if (LOGGER.isLoggable(Level.FINEST)) { + LOGGER.finest("DecideRuleSequence returned " + result); + } + return result; + } +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/ExceedsDocumentLengthThresholdDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/ExceedsDocumentLengthThresholdDecideRule.java new file mode 100644 index 00000000..e7008b49 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/ExceedsDocumentLengthThresholdDecideRule.java @@ -0,0 +1,14 @@ +package org.archive.modules.deciderules; + +public class ExceedsDocumentLengthThresholdDecideRule +extends NotExceedsDocumentLengthTresholdDecideRule { + + + private static final long serialVersionUID = 3L; + + + boolean test(int contentlength) { + return contentlength > getContentLengthThreshold(); + } + +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/ExternalGeoLocationDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/ExternalGeoLocationDecideRule.java new file mode 100644 index 00000000..7b174ef7 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/ExternalGeoLocationDecideRule.java @@ -0,0 +1,131 @@ +/* ExternalGeoLocationDecideRule + * + * Created on May 25, 2005 + * + * Copyright (C) 2005 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.modules.deciderules; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + + +import org.apache.commons.httpclient.URIException; +import org.archive.modules.ProcessorURI; +import org.archive.modules.net.CrawlHost; +import org.archive.modules.net.ServerCache; +import org.springframework.beans.factory.annotation.Autowired; +import org.xbill.DNS.Address; + +/** + * A rule that can be configured to take alternate implementations + * of the ExternalGeoLocationInterface. + * If no implementation specified, or none found, returns configured decision. + * If host in URI has been resolved checks CrawlHost for the country code + * determination. + * If country code is not present, does country lookup, and saves the country + * code to CrawlHost for future consultation. + * If country code is present in CrawlHost, compares it against + * the configured code. + * Note that if a host's IP address changes during the crawl, we still consider + * the associated hostname to be in the country of its original IP address. + * + * @author Igor Ranitovic + */ +public class ExternalGeoLocationDecideRule extends PredicatedDecideRule { + + private static final long serialVersionUID = 3L; + + private static final Logger LOGGER = + Logger.getLogger(ExternalGeoLocationDecideRule.class.getName()); + + ExternalGeoLookupInterface lookup = null; + public ExternalGeoLookupInterface getLookup() { + return this.lookup; + } + public void setLookup(ExternalGeoLookupInterface lookup) { + this.lookup = lookup; + } + + /** + * Country code name. + */ + List countryCodes = new ArrayList(); + public List getCountryCodes() { + return this.countryCodes; + } + public void setCountryCodes(List codes) { + this.countryCodes = codes; + } + + protected ServerCache serverCache; + public ServerCache getServerCache() { + return this.serverCache; + } + @Autowired + public void setServerCache(ServerCache serverCache) { + this.serverCache = serverCache; + } + + public ExternalGeoLocationDecideRule() { + } + + @Override + protected boolean evaluate(ProcessorURI uri) { + ExternalGeoLookupInterface impl = getLookup(); + if (impl == null) { + return false; + } + CrawlHost crawlHost = null; + String host; + InetAddress address; + try { + host = uri.getUURI().getHost(); + crawlHost = serverCache.getHostFor(host); + if (crawlHost.getCountryCode() != null) { + return countryCodes.contains(crawlHost.getCountryCode()); + } + address = crawlHost.getIP(); + if (address == null) { + // TODO: handle transient lookup failures better + address = Address.getByName(host); + } + crawlHost.setCountryCode((String) impl.lookup(address)); + if (countryCodes.contains(crawlHost.getCountryCode())) { + LOGGER.fine("Country Code Lookup: " + " " + host + + crawlHost.getCountryCode()); + return true; + } + } catch (UnknownHostException e) { + LOGGER.log(Level.FINE, "Failed dns lookup " + uri, e); + if (crawlHost != null) { + crawlHost.setCountryCode("--"); + } + } catch (URIException e) { + LOGGER.log(Level.FINE, "Failed to parse hostname " + uri, e); + } + + return false; + } + +} \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/deciderules/ExternalGeoLookupInterface.java b/modules/src/main/java/org/archive/modules/deciderules/ExternalGeoLookupInterface.java new file mode 100644 index 00000000..2814f7c8 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/ExternalGeoLookupInterface.java @@ -0,0 +1,39 @@ +/* ExternalImplInterface + * + * Created on May 25, 2005 + * + * Copyright (C) 2005 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.modules.deciderules; + +import java.net.InetAddress; + +/** + * Interface used by {@link ExternalImplDecideRule}. + * @author stack + * @version $Date$, $Revision$ + * @see ExternalImplDecideRule + */ +public interface ExternalGeoLookupInterface { + /** + * @param obj Object to evaluate + * @return String of a country code that the object belongs to. + */ + public String lookup (InetAddress obj); +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/FetchStatusDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/FetchStatusDecideRule.java new file mode 100644 index 00000000..6661fa93 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/FetchStatusDecideRule.java @@ -0,0 +1,69 @@ +/* FetchStatusDecideRule +* +* $Id$ +* +* Created on Aug 11, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.modules.deciderules; + +import java.util.ArrayList; +import java.util.List; + +import org.archive.modules.ProcessorURI; + +/** + * Rule applies the configured decision for any URI which has a + * fetch status equal to the 'target-status' setting. + * + * @author gojomo + */ +public class FetchStatusDecideRule extends PredicatedDecideRule { + + private static final long serialVersionUID = 3L; + + List statusCodes = new ArrayList(); + public List getStatusCodes() { + return this.statusCodes; + } + public void setStatusCodes(List codes) { + this.statusCodes = codes; + } + + /** + * Default access so available to test code. + */ + static final Integer DEFAULT_TARGET_STATUS = new Integer(0); + + /** + * Usual constructor. + */ + public FetchStatusDecideRule() { + } + + /** + * Evaluate whether given object is equal to the configured status + */ + @Override + protected boolean evaluate(ProcessorURI uri) { + return getStatusCodes().contains(uri.getFetchStatus()); + } + +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/FetchStatusMatchesRegExpDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/FetchStatusMatchesRegExpDecideRule.java new file mode 100644 index 00000000..f3a4e7b2 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/FetchStatusMatchesRegExpDecideRule.java @@ -0,0 +1,45 @@ +/* $Id$ +* +* Created on Sep 4, 2006 +* +* Copyright (C) 2006 Olaf Freyer. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.modules.deciderules; + + +import org.archive.modules.ProcessorURI; + + + +public class FetchStatusMatchesRegExpDecideRule extends MatchesRegExpDecideRule { + + private static final long serialVersionUID = 3L; + + + /** + * Usual constructor. + */ + public FetchStatusMatchesRegExpDecideRule() { + } + + + protected String getString(ProcessorURI uri) { + return Integer.toString(uri.getFetchStatus()); + } +} \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/deciderules/FetchStatusNotMatchesRegExpDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/FetchStatusNotMatchesRegExpDecideRule.java new file mode 100644 index 00000000..8ffc8103 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/FetchStatusNotMatchesRegExpDecideRule.java @@ -0,0 +1,52 @@ +/* $Id: FetchStatusNotMatchesRegExpDecideRule.java 4649 2006-09-25 17:16:55Z paul_jack $ +* +* Created on Sep 4, 2006 +* +* Copyright (C) 2006 Olaf Freyer. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.modules.deciderules; + +import org.archive.modules.ProcessorURI; + + +public class FetchStatusNotMatchesRegExpDecideRule +extends FetchStatusMatchesRegExpDecideRule { + + private static final long serialVersionUID = -2220182698344063577L; +// private final Logger logger = Logger.getLogger(this.getClass().getName()); + + /** + * Usual constructor. + * @param name + */ + public FetchStatusNotMatchesRegExpDecideRule() { + } + + /** + * Evaluate whether given object's FetchStatus does not match + * configured regexp (by reversing the superclass's answer). + * + * @param object Object to make decision about. + * @return true if the regexp is not matched + */ + @Override + protected boolean evaluate(ProcessorURI object) { + return ! super.evaluate(object); + } +} \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/deciderules/HasViaDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/HasViaDecideRule.java new file mode 100644 index 00000000..1542e549 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/HasViaDecideRule.java @@ -0,0 +1,54 @@ +/* HasViaDecideRule +* +* $Id$ +* +* Created on Aug 11, 2006 +* +* Copyright (C) 2006 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.modules.deciderules; + +import org.archive.modules.ProcessorURI; + +/** + * Rule applies the configured decision for any URI which has a 'via' + * (essentially, any URI that was a seed or some kinds of mid-crawl adds). + * + * @author gojomo + */ +public class HasViaDecideRule extends PredicatedDecideRule { + + private static final long serialVersionUID = 3L; + + /** + * Usual constructor. + * @param name Name of this DecideRule. + */ + public HasViaDecideRule() { + } + + /** + * Evaluate whether given object is over the threshold number of + * hops. + */ + @Override + protected boolean evaluate(ProcessorURI uri) { + return uri.getVia() != null; + } +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/HopsPathMatchesRegExpDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/HopsPathMatchesRegExpDecideRule.java new file mode 100644 index 00000000..2d672de0 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/HopsPathMatchesRegExpDecideRule.java @@ -0,0 +1,50 @@ +/* HopsPathMatchesRegExpDecideRule +* +* $Id$ +* +* Created on June 23, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.modules.deciderules; + +import org.archive.modules.ProcessorURI; + +/** + * Rule applies configured decision to any CrawlURIs whose 'hops-path' + * (string like "LLXE" etc.) matches the supplied regexp. + * + * @author gojomo + */ +public class HopsPathMatchesRegExpDecideRule extends MatchesRegExpDecideRule { + private static final long serialVersionUID = 3L; + + /** + * Usual constructor. + * @param name + */ + public HopsPathMatchesRegExpDecideRule() { + } + + + @Override + protected String getString(ProcessorURI uri) { + return uri.getPathFromSeed(); + } +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/MatchesFilePatternDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/MatchesFilePatternDecideRule.java new file mode 100644 index 00000000..e471b27e --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/MatchesFilePatternDecideRule.java @@ -0,0 +1,107 @@ +/* + * MatchesFilePatternDecideRule + * + * $Id$ + * + * Created on Mar 11, 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +package org.archive.modules.deciderules; + +import java.util.regex.Pattern; + +/** + * Compares suffix of a passed CrawlURI, UURI, or String against a regular + * expression pattern, applying its configured decision to all matches. + * + * Several predefined patterns are available for convenience. Choosing + * 'custom' makes this the same as a regular MatchesRegExpDecideRule. + * + * @author Igor Ranitovic + */ +public class MatchesFilePatternDecideRule extends MatchesRegExpDecideRule { + + public static enum Preset { + + + ALL(".*(?i)(\\.(bmp|gif|jpe?g|png|tiff?|mid|mp2|mp3|mp4|wav|avi|mov" + + "|mpeg|ram|rm|smil|wmv|doc|pdf|ppt|swf))$"), + + IMAGES(".*(?i)(\\.(bmp|gif|jpe?g|png|tiff?))$"), + + AUDIO(".*(?i)(\\.(mid|mp2|mp3|mp4|wav))$"), + + VIDEO(".*(?i)(\\.(avi|mov|mpeg|ram|rm|smil|wmv))$"), + + MISC(".*(?i)(\\.(doc|pdf|ppt|swf))$"), + + CUSTOM(null); + + final private Pattern pattern; + + Preset(String regexp) { + if (regexp == null) { + pattern = null; + } else { + pattern = Pattern.compile(regexp); + } + } + + + public Pattern getPattern() { + return pattern; + } + } + + private static final long serialVersionUID = 3L; + + { + setUsePreset(Preset.ALL); + } + public Preset getUsePreset() { + return (Preset) kp.get("usePreset"); + } + public void setUsePreset(Preset preset) { + kp.put("usePreset",preset); + } + + /** + * Usual constructor. + */ + public MatchesFilePatternDecideRule() { + } + + /** + * Use a preset if configured to do so. + * @param o Context + * @return Regex to use. + * + * @see org.archive.crawler.filter.URIRegExpFilter#getRegexp(Object) + */ + @Override + public Pattern getRegex() { + Preset preset = getUsePreset(); + if (preset == Preset.CUSTOM) { + return getRegex(); + } + return preset.getPattern(); + } +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/MatchesListRegExpDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/MatchesListRegExpDecideRule.java new file mode 100644 index 00000000..63ad2504 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/MatchesListRegExpDecideRule.java @@ -0,0 +1,132 @@ +/* MatchesListRegExpDecideRule + * + * $Id$ + * + * Created on 30.5.2005 + * + * Copyright (C) 2005 Kristinn Sigurdsson + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.modules.deciderules; + +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Pattern; + +import org.archive.modules.ProcessorURI; + +/** + * Rule applies configured decision to any CrawlURIs whose String URI + * matches the supplied regexps. + *

      + * The list of regular expressions can be considered logically AND or OR. + * + * @author Kristinn Sigurdsson + * + * @see MatchesRegExpDecideRule + */ +public class MatchesListRegExpDecideRule extends PredicatedDecideRule { + + + private static final long serialVersionUID = 3L; + + private static final Logger logger = + Logger.getLogger(MatchesListRegExpDecideRule.class.getName()); + + + /** + * The list of regular expressions to evalute against the URI. + */ + { + setRegexList(new ArrayList()); + } + @SuppressWarnings("unchecked") + public List getRegexList() { + return (List) kp.get("regexList"); + } + public void setRegexList(List patterns) { + kp.put("regexList", patterns); + } + + /** + * True if the list of regular expression should be considered as logically + * AND when matching. False if the list of regular expressions should be + * considered as logically OR when matching. + */ + { + setListLogicalOr(true); + } + public boolean getListLogicalOr() { + return (Boolean) kp.get("listLogicalOr"); + } + public void setListLogicalOr(boolean listLogicalOr) { + kp.put("listLogicalOr",listLogicalOr); + } + + /** + * Usual constructor. + */ + public MatchesListRegExpDecideRule() { + } + + /** + * Evaluate whether given object's string version + * matches configured regexps + */ + @Override + protected boolean evaluate(ProcessorURI uri) { + List regexps = getRegexList(); + if(regexps.size()==0){ + return false; + } + + String str = uri.toString(); + boolean listLogicOR = getListLogicalOr(); + + for (Pattern p: regexps) { + boolean matches = p.matcher(str).matches(); + + if (logger.isLoggable(Level.FINER)) { + logger.finer("Tested '" + str + "' match with regex '" + + p.pattern() + " and result was " + matches); + } + + if(matches){ + if(listLogicOR){ + // OR based and we just got a match, done! + logger.fine("Matched: " + str); + return true; + } + } else { + if(listLogicOR == false){ + // AND based and we just found a non-match, done! + return false; + } + } + } + + if (listLogicOR) { + return false; + } else { + return true; + } + } + +} \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/deciderules/MatchesRegExpDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/MatchesRegExpDecideRule.java new file mode 100644 index 00000000..6dc4825d --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/MatchesRegExpDecideRule.java @@ -0,0 +1,74 @@ +/* MatchesRegExpDecideRule +* +* $Id$ +* +* Created on Apr 4, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.modules.deciderules; + +import java.util.regex.Pattern; + +import org.archive.modules.ProcessorURI; + +/** + * Rule applies configured decision to any ProcessorURIs whose String URI + * matches the supplied regexp. + * + * @author gojomo + */ +public class MatchesRegExpDecideRule extends PredicatedDecideRule { + + private static final long serialVersionUID = 2L; + + { + setRegex(Pattern.compile(".")); + } + public Pattern getRegex() { + return (Pattern) kp.get("regex"); + } + public void setRegex(Pattern regex) { + kp.put("regex",regex); + } + + /** + * Usual constructor. + */ + public MatchesRegExpDecideRule() { + } + + + /** + * Evaluate whether given object's string version + * matches configured regexp + * + * @param object + * @return true if regexp is matched + */ + @Override + protected boolean evaluate(ProcessorURI uri) { + Pattern p = getRegex(); + return p.matcher(getString(uri)).matches(); + } + + protected String getString(ProcessorURI uri) { + return uri.toString(); + } +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/NotExceedsDocumentLengthTresholdDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/NotExceedsDocumentLengthTresholdDecideRule.java new file mode 100644 index 00000000..bb02d809 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/NotExceedsDocumentLengthTresholdDecideRule.java @@ -0,0 +1,131 @@ +/* $Id: NotExceedsDocumentLengthTresholdDecideRule.java 4649 2006-09-25 17:16:55Z paul_jack $ + * + * Created on 28.8.2006 + * + * Copyright (C) 2006 Olaf Freyer + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.modules.deciderules; + +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.httpclient.HttpMethod; +import org.archive.modules.ProcessorURI; + +/** + * Applies configured decision for URIs with content length less than a given + * treshold. Either examines HTTP header content length or actual downloaded + * content length and returns false for documents exceeding a given length + * treshold. + */ +public class NotExceedsDocumentLengthTresholdDecideRule +extends PredicatedDecideRule { + + + private static final long serialVersionUID = -8774160016195991876L; + + private static final Logger logger = Logger. + getLogger(NotExceedsDocumentLengthTresholdDecideRule.class.getName()); + + + /** + * Shall this rule be used as a midfetch rule? If true, this rule will + * determine content length based on HTTP header information, otherwise + * the size of the already downloaded content will be used. + */ + { + setUseHeaderLength(true); + } + public boolean getUseHeaderLength() { + return (Boolean) kp.get("useHeaderLength"); + } + public void setUseHeaderLength(boolean useHeaderLength) { + kp.put("useHeaderLength",useHeaderLength); + } + + /** + * Max content-length this filter will allow to pass through. If -1, + * then no limit. + */ + { + setContentLengthThreshold(-1L); + } + public long getContentLengthThreshold() { + return (Long) kp.get("contentLengthThreshold"); + } + public void setContentLengthThreshold(long threshold) { + kp.put("contentLengthThreshold",threshold); + } + + // Header predictor state constants + public static final int HEADER_PREDICTS_MISSING = -1; + + public NotExceedsDocumentLengthTresholdDecideRule() { + } + + protected boolean evaluate(ProcessorURI curi) { + int contentlength = HEADER_PREDICTS_MISSING; + + // filter used as midfetch filter + if (getUseHeaderLength()) { + + if (curi.getHttpMethod() == null) { + // Missing header info, let pass + if (logger.isLoggable(Level.INFO)) { + logger.info("Error: Missing HttpMethod object in " + + "CrawlURI. " + curi.toString()); + } + return false; + } + + // Initially assume header info is missing + HttpMethod method = curi.getHttpMethod(); + + // get content-length + String newContentlength = null; + if (method.getResponseHeader("content-length") != null) { + newContentlength = method.getResponseHeader("content-length") + .getValue(); + } + + if (newContentlength != null && newContentlength.length() > 0) { + try { + contentlength = Integer.parseInt(newContentlength); + } catch (NumberFormatException nfe) { + // Ignore. + } + } + + // If no document length was reported or format was wrong, + // let pass + if (contentlength == HEADER_PREDICTS_MISSING) { + return false; + } + } else { + contentlength = (int) curi.getContentSize(); + } + + return test(contentlength); + } + + + boolean test(int contentlength) { + return contentlength < getContentLengthThreshold(); + } +} \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/deciderules/NotMatchesFilePatternDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/NotMatchesFilePatternDecideRule.java new file mode 100644 index 00000000..9d553537 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/NotMatchesFilePatternDecideRule.java @@ -0,0 +1,61 @@ +/* NotMatchesFilePatternDecideRule +* +* $Id: NotMatchesFilePatternDecideRule.java 4649 2006-09-25 17:16:55Z paul_jack $ +* +* Created on Apr 4, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.modules.deciderules; + +import org.archive.modules.ProcessorURI; + + +/** + * Rule applies configured decision to any URIs which do *not* + * match the supplied (file-pattern) regexp. + * + * @author gojomo + */ +public class NotMatchesFilePatternDecideRule +extends MatchesFilePatternDecideRule { + + private static final long serialVersionUID = -8161371026787859554L; + //private static final Logger logger = + // Logger.getLogger(NotMatchesRegExpDecideRule.class.getName()); + + /** + * Usual constructor. + * @param name + */ + public NotMatchesFilePatternDecideRule() { + } + + /** + * Evaluate whether given object's string version does not match + * configured regexp (by reversing the superclass's answer). + * + * @param object Object to make decision about. + * @return true if the regexp is not matched + */ + @Override + protected boolean evaluate(ProcessorURI uri) { + return ! super.evaluate(uri); + } +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/NotMatchesListRegExpDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/NotMatchesListRegExpDecideRule.java new file mode 100644 index 00000000..4dd26a87 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/NotMatchesListRegExpDecideRule.java @@ -0,0 +1,62 @@ +/* NotMatchesListRegExpDecideRule + * + * $Id: NotMatchesListRegExpDecideRule.java 4721 2006-11-14 20:03:18Z stack-sf $ + * + * Created on 30.5.2005 + * + * Copyright (C) 2005 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.modules.deciderules; + +import org.archive.modules.ProcessorURI; + + +/** + * Rule applies configured decision to any URIs which do *not* + * match the supplied regexp. + * + * @author Kristinn Sigurdsson + */ +public class NotMatchesListRegExpDecideRule extends MatchesListRegExpDecideRule { + + private static final long serialVersionUID = 8691360087063555583L; + + //private static final Logger logger = + // Logger.getLogger(NotMatchesListRegExpDecideRule.class.getName()); + + + /** + * Usual constructor. + * @param name + */ + public NotMatchesListRegExpDecideRule() { + } + + /** + * Evaluate whether given object's string version does not match + * configured regexps (by reversing the superclass's answer). + * + * @param object Object to make decision about. + * @return true if the regexps are not matched + */ + @Override + protected boolean evaluate(ProcessorURI object) { + return ! super.evaluate(object); + } +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/NotMatchesRegExpDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/NotMatchesRegExpDecideRule.java new file mode 100644 index 00000000..242ac87d --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/NotMatchesRegExpDecideRule.java @@ -0,0 +1,60 @@ +/* NotMatchesRegExpDecideRule +* +* $Id: NotMatchesRegExpDecideRule.java 4649 2006-09-25 17:16:55Z paul_jack $ +* +* Created on Apr 4, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.modules.deciderules; + +import org.archive.modules.ProcessorURI; + + +/** + * Rule applies configured decision to any URIs which do *not* + * match the supplied regexp. + * + * @author gojomo + */ +public class NotMatchesRegExpDecideRule extends MatchesRegExpDecideRule { + + private static final long serialVersionUID = -2085313401991694306L; + //private static final Logger logger = + // Logger.getLogger(NotMatchesRegExpDecideRule.class.getName()); + + + /** + * Usual constructor. + */ + public NotMatchesRegExpDecideRule(String name) { + } + + /** + * Evaluate whether given object's string version does not match + * configured regexp (by reversing the superclass's answer). + * + * @param object Object to make decision about. + * @return true if the regexp is not matched + */ + @Override + protected boolean evaluate(ProcessorURI object) { + return ! super.evaluate(object); + } +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/PathologicalPathDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/PathologicalPathDecideRule.java new file mode 100644 index 00000000..493e1267 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/PathologicalPathDecideRule.java @@ -0,0 +1,100 @@ +/* PathologicalPathDecideRule +* +* $Id$ +* +* Created on Apr 1, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.modules.deciderules; + +import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Pattern; + +import org.archive.modules.ProcessorURI; + + +/** + * Rule REJECTs any URI which contains an excessive number of identical, + * consecutive path-segments (eg http://example.com/a/a/a/boo.html == 3 '/a' + * segments) + * + * @author gojomo + */ +public class PathologicalPathDecideRule extends DecideRule { + + private static final long serialVersionUID = 3L; + + /** + * Number of times the pattern should be allowed to occur. This rule returns + * its decision (usually REJECT) if a path-segment is repeated more than + * number of times. + */ + { + setMaxRepetitions(2); + } + public int getMaxRepetitions() { + return (Integer) kp.get("maxRepetitions"); + } + public void setMaxRepetitions(int maxRepetitions) { + kp.put("maxRepetitions", maxRepetitions); + } + + private AtomicReference pattern = new AtomicReference(); + + /** Constructs a new PathologicalPathFilter. + * + * @param name the name of the filter. + */ + public PathologicalPathDecideRule() { + } + + + @Override + protected DecideResult innerDecide(ProcessorURI uri) { + int maxRep = getMaxRepetitions(); + Pattern p = getPattern(maxRep); + if (p.matcher(uri.getUURI().toString()).matches()) { + return DecideResult.REJECT; + } else { + return DecideResult.PASS; + } + } + + /** + * Construct the regexp string to be matched against the URI. + * @param o an object to extract a URI from. + * @return the regexp pattern. + */ + private Pattern getPattern(int maxRep) { + // race no concern: assignment is atomic, happy with any last value + Pattern p = pattern.get(); + if (p != null) { + return p; + } + String regex = constructRegexp(maxRep); + p = Pattern.compile(regex); + pattern.set(p); + return p; + } + + protected String constructRegexp(int rep) { + return (rep == 0) ? null : ".*?/(.*?/)\\1{" + rep + ",}.*"; + } +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/PredicatedDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/PredicatedDecideRule.java new file mode 100644 index 00000000..d99ee5f7 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/PredicatedDecideRule.java @@ -0,0 +1,60 @@ +/* AcceptRule +* +* $Id: PredicatedDecideRule.java 4914 2007-02-18 21:53:01Z gojomo $ +* +* Created on Mar 3, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.modules.deciderules; + +import org.archive.modules.ProcessorURI; + +/** + * Rule which applies the configured decision only if a + * test evaluates to true. Subclasses override evaluate() + * to establish the test. + * + * @author gojomo + */ +public abstract class PredicatedDecideRule extends DecideRule { + + { + setDecision(DecideResult.ACCEPT); + } + public DecideResult getDecision() { + return (DecideResult) kp.get("decision"); + } + public void setDecision(DecideResult decision) { + kp.put("decision",decision); + } + + public PredicatedDecideRule() { + } + + @Override + protected DecideResult innerDecide(ProcessorURI uri) { + if (evaluate(uri)) { + return getDecision(); + } + return DecideResult.PASS; + } + + protected abstract boolean evaluate(ProcessorURI object); +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/PrerequisiteAcceptDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/PrerequisiteAcceptDecideRule.java new file mode 100644 index 00000000..630af93f --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/PrerequisiteAcceptDecideRule.java @@ -0,0 +1,56 @@ +/* AcceptDecideRule +* +* $Id$ +* +* Created on Mar 3, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.modules.deciderules; + +import org.archive.modules.ProcessorURI; +import org.archive.modules.extractor.Hop; + + +/** + * Rule which ACCEPTs all 'prerequisite' URIs (those with a 'P' in + * the last hopsPath position). Good in a late position to ensure + * other scope settings don't lock out necessary prerequisites. + * + * @author gojomo + */ +public class PrerequisiteAcceptDecideRule extends DecideRule { + + private static final long serialVersionUID = 3L; + + public PrerequisiteAcceptDecideRule() { + } + + + public DecideResult innerDecide(ProcessorURI uri) { + String hopsPath = uri.getPathFromSeed(); + if (hopsPath != null && hopsPath.length() > 0 && + hopsPath.charAt(hopsPath.length()-1) == Hop.PREREQ.getHopChar()) { + return DecideResult.ACCEPT; + } + return DecideResult.PASS; + } + + +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/RejectDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/RejectDecideRule.java new file mode 100644 index 00000000..0978a8a6 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/RejectDecideRule.java @@ -0,0 +1,44 @@ +/* Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * RejectDecideRule.java + * Created on October 5, 2006 + * + * $Header$ + */ +package org.archive.modules.deciderules; + +import org.archive.modules.ProcessorURI; + + +public class RejectDecideRule extends DecideRule { + + private static final long serialVersionUID = 3L; + + + @Override + protected DecideResult innerDecide(ProcessorURI uri) { + return DecideResult.REJECT; + } + + + @Override + public DecideResult onlyDecision(ProcessorURI uri) { + return DecideResult.REJECT; + } +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/SeedAcceptDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/SeedAcceptDecideRule.java new file mode 100644 index 00000000..73060e9f --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/SeedAcceptDecideRule.java @@ -0,0 +1,53 @@ +/* SeedAcceptDecideRule +* +* $Id$ +* +* Created on Sep 13, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.modules.deciderules; + +import org.archive.modules.ProcessorURI; + + +/** + * Rule which ACCEPTs all 'seed' URIs (those for which + * isSeed is true). Good in a late position to ensure + * other scope settings don't lock out explicitly added + * seeds. + * + * @author gojomo + */ +public class SeedAcceptDecideRule extends DecideRule { + + private static final long serialVersionUID = 3L; + + public SeedAcceptDecideRule() { + } + + + @Override + protected DecideResult innerDecide(ProcessorURI uri) { + if (uri.isSeed()) { + return DecideResult.ACCEPT; + } + return DecideResult.PASS; + } +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/TooManyHopsDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/TooManyHopsDecideRule.java new file mode 100644 index 00000000..9363a983 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/TooManyHopsDecideRule.java @@ -0,0 +1,85 @@ +/* AcceptRule +* +* $Id$ +* +* Created on Apr 1, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.modules.deciderules; + +import org.archive.modules.ProcessorURI; + + + +/** + * Rule REJECTs any CrawlURIs whose total number of hops (length of the + * hopsPath string, traversed links of any type) is over a threshold. + * Otherwise returns PASS. + * + * @author gojomo + */ +public class TooManyHopsDecideRule extends PredicatedDecideRule { + + private static final long serialVersionUID = 3L; + + /** default for this class is to REJECT */ + { + setDecision(DecideResult.REJECT); + } + + /** + * Max path depth for which this filter will match. + */ + { + setMaxHops(20); + } + public int getMaxHops() { + return (Integer) kp.get("maxHops"); + } + public void setMaxHops(int maxHops) { + kp.put("maxHops", maxHops); + } + + /** + * Usual constructor. + */ + public TooManyHopsDecideRule() { + } + + /** + * Evaluate whether given object is over the threshold number of + * hops. + * + * @param object + * @return true if the mx-hops is exceeded + */ + @Override + protected boolean evaluate(ProcessorURI uri) { + String hops = uri.getPathFromSeed(); + if (hops == null) { + return false; + } + if (hops.length() <= getMaxHops()) { + return false; + } + return true; + } + +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/TooManyPathSegmentsDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/TooManyPathSegmentsDecideRule.java new file mode 100644 index 00000000..13764577 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/TooManyPathSegmentsDecideRule.java @@ -0,0 +1,86 @@ +/* AcceptRule +* +* $Id$ +* +* Created on Apr 1, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.modules.deciderules; + +import org.archive.modules.ProcessorURI; + +/** + * Rule REJECTs any CrawlURIs whose total number of path-segments (as + * indicated by the count of '/' characters not including the first '//') + * is over a given threshold. + * + * @author gojomo + */ +public class TooManyPathSegmentsDecideRule extends PredicatedDecideRule { + + private static final long serialVersionUID = 3L; + + /** default for this class is to REJECT */ + { + setDecision(DecideResult.REJECT); + } + + /** + * Number of path segments beyond which this rule will reject URIs. + */ + { + setMaxPathDepth(20); + } + public int getMaxPathDepth() { + return (Integer) kp.get("maxPathDepth"); + } + public void setMaxPathDepth(int maxPathDepth) { + kp.put("maxPathDepth", maxPathDepth); + } + /** + * Usual constructor. + */ + public TooManyPathSegmentsDecideRule() { + } + + /** + * Evaluate whether given object is over the threshold number of + * path-segments. + * + * @param object + * @return true if the path-segments is exceeded + */ + @Override + protected boolean evaluate(ProcessorURI curi) { + String uri = curi.toString(); + int count = 0; + int threshold = getMaxPathDepth(); + for (int i = 0; i < uri.length(); i++) { + if (uri.charAt(i) == '/') { + count++; + } + if (count > threshold) { + return true; + } + } + return false; + } + +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/TransclusionDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/TransclusionDecideRule.java new file mode 100644 index 00000000..3eff2784 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/TransclusionDecideRule.java @@ -0,0 +1,122 @@ +/* TransclusionDecideRule +* +* $Id$ +* +* Created on Apr 1, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.modules.deciderules; + +import org.archive.modules.ProcessorURI; +import org.archive.modules.extractor.Hop; + +/** + * Rule ACCEPTs any CrawlURIs whose path-from-seed ('hopsPath' -- see + * {@link CandidateURI#getPathFromSeed()}) ends + * with at least one, but not more than, the given number of + * non-navlink ('L') hops. + * + * Otherwise, if the path-from-seed is empty or if a navlink ('L') occurs + * within max-trans-hops of the tail of the path-from-seed, this rule + * returns PASS. + * + *

      Thus, it allows things like embedded resources (frames/images/media) + * and redirects to be transitively included ('transcluded') in a crawl, + * even if they otherwise would not, for some reasonable number of hops + * (1-4). + * + * @see Transclusion + * + * @author gojomo + */ +public class TransclusionDecideRule extends PredicatedDecideRule { + + private static final long serialVersionUID = -3975688876990558918L; + + /** + * Maximum number of non-navlink (non-'L') hops to ACCEPT. + */ + { + setMaxTransHops(2); + } + public int getMaxTransHops() { + return (Integer) kp.get("maxTransHops"); + } + public void setMaxTransHops(int maxTransHops) { + kp.put("maxTransHops", maxTransHops); + } + + /** + * Maximum number of speculative ('X') hops to ACCEPT. + */ + { + setMaxSpeculativeHops(1); + } + public int getMaxSpeculativeHops() { + return (Integer) kp.get("maxSpeculativeHops"); + } + public void setMaxSpeculativeHops(int maxSpeculativeHops) { + kp.put("maxSpeculativeHops", maxSpeculativeHops); + } + + /** + * Usual constructor. + */ + public TransclusionDecideRule() { + } + + /** + * Evaluate whether given object is within the threshold number of + * transitive hops. + * + * @param object Object to make decision on. + * @return true if the transitive hops >0 and <= max + */ + protected boolean evaluate(ProcessorURI curi) { + String hopsPath = curi.getPathFromSeed(); + if (hopsPath == null || hopsPath.length() == 0) { + return false; + } + int count = 0; + int specCount = 0; + for (int i = hopsPath.length() - 1; i >= 0; i--) { + char c = hopsPath.charAt(i); + if (c != Hop.NAVLINK.getHopChar()) { + count++; + if(c == Hop.SPECULATIVE.getHopChar()) { + specCount++; + } + } else { + break; + } + } + if (count <= 0) { + return false; + } + + if (specCount > getMaxSpeculativeHops()) { + return false; + } + + return count <= getMaxTransHops(); + } + + +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/recrawl/IdenticalDigestDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/recrawl/IdenticalDigestDecideRule.java new file mode 100644 index 00000000..d225b238 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/recrawl/IdenticalDigestDecideRule.java @@ -0,0 +1,88 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.modules.deciderules.recrawl; + +import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_CONTENT_DIGEST; +import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY; + +import java.util.Map; + +import org.archive.modules.ProcessorURI; +import org.archive.modules.deciderules.DecideResult; +import org.archive.modules.deciderules.PredicatedDecideRule; + +/** + * Rule applies configured decision to any CrawlURIs whose prior-history + * content-digest matches the latest fetch. + * + * @author gojomo + */ +public class IdenticalDigestDecideRule extends PredicatedDecideRule { + private static final long serialVersionUID = 4275993790856626949L; + + /** default for this class is to REJECT */ + { + setDecision(DecideResult.REJECT); + } + + /** + * Usual constructor. + */ + public IdenticalDigestDecideRule() { + } + + /** + * Evaluate whether given CrawlURI's content-digest exactly + * matches that of preceding fetch. + * + * @param object should be CrawlURI + * @return true if current-fetch content-digest matches previous + */ + protected boolean evaluate(ProcessorURI curi) { + return hasIdenticalDigest(curi); + } + + + /** + * Utility method for testing if a CrawlURI's last two history + * entiries (one being the most recent fetch) have identical + * content-digest information. + * + * @param curi CrawlURI to test + * @return true if last two history entries have identical digests, + * otherwise false + */ + @SuppressWarnings("unchecked") + public static boolean hasIdenticalDigest(ProcessorURI curi) { + if(curi.containsDataKey(A_FETCH_HISTORY)) { + Map[] history = + (Map[])curi.getData().get(A_FETCH_HISTORY); + return history[0] != null + && history[0].containsKey(A_CONTENT_DIGEST) + && history[1] != null + && history[1].containsKey(A_CONTENT_DIGEST) + && history[0].get(A_CONTENT_DIGEST).equals( + history[1].get(A_CONTENT_DIGEST)); + } else { + return false; + } + } + +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/surt/NotOnDomainsDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/surt/NotOnDomainsDecideRule.java new file mode 100644 index 00000000..be367945 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/surt/NotOnDomainsDecideRule.java @@ -0,0 +1,61 @@ +/* NotOnDomainsDecideRule +* +* $Id: NotOnDomainsDecideRule.java 4649 2006-09-25 17:16:55Z paul_jack $ +* +* Created on Apr 5, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.modules.deciderules.surt; + +import org.archive.modules.ProcessorURI; + + +/** + * Rule applies configured decision to any URIs that are + * *not* in one of the domains in the configured set of + * domains, filled from the seed set. + * + * @author gojomo + */ +public class NotOnDomainsDecideRule extends OnDomainsDecideRule { + + private static final long serialVersionUID = -1634035244888724934L; + + //private static final Logger logger = + // Logger.getLogger(NotOnDomainsDecideRule.class.getName()); + /** + * Usual constructor. + * @param name + */ + public NotOnDomainsDecideRule() { + } + + /** + * Evaluate whether given object's URI is NOT in the set of + * domains -- simply reverse superclass's determination + * + * @param object to evaluate + * @return true if URI is not in domain set + */ + protected boolean evaluate(ProcessorURI object) { + boolean superDecision = super.evaluate(object); + return !superDecision; + } +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/surt/NotOnHostsDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/surt/NotOnHostsDecideRule.java new file mode 100644 index 00000000..46969f7b --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/surt/NotOnHostsDecideRule.java @@ -0,0 +1,60 @@ +/* NotOnHostsDecideRule +* +* $Id: NotOnHostsDecideRule.java 4649 2006-09-25 17:16:55Z paul_jack $ +* +* Created on Apr 5, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.modules.deciderules.surt; + +import org.archive.modules.ProcessorURI; + + +/** + * Rule applies configured decision to any URIs that + * are *not* on one of the hosts in the configured set of + * hosts, filled from the seed set. + * + * @author gojomo + */ +public class NotOnHostsDecideRule extends OnHostsDecideRule { + + private static final long serialVersionUID = 1512825197255050412L; + + + /** + * Usual constructor. + */ + public NotOnHostsDecideRule() { + } + + /** + * Evaluate whether given object's URI is NOT in the set of + * hosts -- simply reverse superclass's determination + * + * @param object Object to evaluate + * @return true if URI not in set + */ + @Override + protected boolean evaluate(ProcessorURI object) { + boolean superDecision = super.evaluate(object); + return !superDecision; + } +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/surt/NotSurtPrefixedDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/surt/NotSurtPrefixedDecideRule.java new file mode 100644 index 00000000..0855bfb9 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/surt/NotSurtPrefixedDecideRule.java @@ -0,0 +1,63 @@ +/* NotSurtPrefixedDecideRule +* +* $Id: NotSurtPrefixedDecideRule.java 4649 2006-09-25 17:16:55Z paul_jack $ +* +* Created on Apr 5, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.modules.deciderules.surt; + +import org.archive.modules.ProcessorURI; + + +/** + * Rule applies configured decision to any URIs that, when + * expressed in SURT form, do *not* begin with one of the prefixes + * in the configured set. + * + * The set can be filled with SURT prefixes implied or + * listed in the seeds file, or another external file. + * + * @author gojomo + */ +public class NotSurtPrefixedDecideRule extends SurtPrefixedDecideRule { + + private static final long serialVersionUID = -7491388438128566377L; + + //private static final Logger logger = + // Logger.getLogger(NotSurtPrefixedDecideRule.class.getName()); + /** + * Usual constructor. + * @param name + */ + public NotSurtPrefixedDecideRule() { + } + + /** + * Evaluate whether given object's URI is NOT in the SURT + * prefix set -- simply reverse superclass's determination + * + * @param object + * @return true if regexp is matched + */ + protected boolean evaluate(ProcessorURI object) { + return !super.evaluate(object); + } +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/surt/OnDomainsDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/surt/OnDomainsDecideRule.java new file mode 100644 index 00000000..cf5fad2f --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/surt/OnDomainsDecideRule.java @@ -0,0 +1,62 @@ +/* OnDomainsDecideRule +* +* $Id$ +* +* Created on Apr 5, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.modules.deciderules.surt; + +import org.archive.util.SurtPrefixSet; + +/** + * Rule applies configured decision to any URIs that + * are on one of the domains in the configured set of + * domains, filled from the seed set. + * + * @author gojomo + */ +public class OnDomainsDecideRule extends SurtPrefixedDecideRule { + + private static final long serialVersionUID = 3L; + //private static final Logger logger = + // Logger.getLogger(OnDomainsDecideRule.class.getName()); + /** + * Usual constructor. + */ + public OnDomainsDecideRule() { + super(); + } + + /** + * Patch the SURT prefix set so that it only includes host-enforcing prefixes + * + * @see org.archive.modules.deciderules.surt.SurtPrefixedDecideRule#readPrefixes() + */ + protected void readPrefixes(/*StateProvider context*/) { + buildSurtPrefixSet(); + surtPrefixes.convertAllPrefixesToDomains(); + dumpSurtPrefixSet(); + } + + protected String prefixFrom(String uri) { + return SurtPrefixSet.convertPrefixToDomain(super.prefixFrom(uri)); + } +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/surt/OnHostsDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/surt/OnHostsDecideRule.java new file mode 100644 index 00000000..ff594ab8 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/surt/OnHostsDecideRule.java @@ -0,0 +1,64 @@ +/* OnHostsDecideRule +* +* $Id$ +* +* Created on Apr 5, 2005 +* +* Copyright (C) 2005 Internet Archive. +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.modules.deciderules.surt; + +import org.archive.util.SurtPrefixSet; + +/** + * Rule applies configured decision to any URIs that + * are on one of the hosts in the configured set of + * hosts, filled from the seed set. + * + * @author gojomo + */ +public class OnHostsDecideRule extends SurtPrefixedDecideRule { + + private static final long serialVersionUID = 3L; + + //private static final Logger logger = + // Logger.getLogger(OnHostsDecideRule.class.getName()); + /** + * Usual constructor. + * @param name + */ + public OnHostsDecideRule() { + super(); + } + + /** + * Patch the SURT prefix set so that it only includes host-enforcing prefixes + * + * @see org.archive.modules.deciderules.surt.SurtPrefixedDecideRule#readPrefixes() + */ + protected void readPrefixes(/*StateProvider context*/) { + buildSurtPrefixSet(); + surtPrefixes.convertAllPrefixesToHosts(); + dumpSurtPrefixSet(); + } + + protected String prefixFrom(String uri) { + return SurtPrefixSet.convertPrefixToHost(super.prefixFrom(uri)); + } +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/surt/ScopePlusOneDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/surt/ScopePlusOneDecideRule.java new file mode 100644 index 00000000..4b767207 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/surt/ScopePlusOneDecideRule.java @@ -0,0 +1,155 @@ +/* ScopePlusOneDecideRule +* +* Created on Aug 22, 2005 +* +* Copyright 2005 Regents of the University of California, All rights reserved +* +* This file is part of the Heritrix web crawler (crawler.archive.org). +* +* Heritrix is free software; you can redistribute it and/or modify +* it under the terms of the GNU Lesser Public License as published by +* the Free Software Foundation; either version 2.1 of the License, or +* any later version. +* +* Heritrix is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Lesser Public License for more details. +* +* You should have received a copy of the GNU Lesser Public License +* along with Heritrix; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package org.archive.modules.deciderules.surt; + +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.archive.modules.ProcessorURI; +import org.archive.modules.deciderules.DecideResult; +import org.archive.net.UURI; +import org.archive.util.SurtPrefixSet; + +/** + * Rule allows one level of discovery beyond configured scope + * (e.g. Domain, plus the first otherwise out-of-scope link from an + * in-scope page, but not further hops from that first page) + * + * @author Shifra Raffel + * @version $Date$ $Revision$ + */ +public class ScopePlusOneDecideRule extends SurtPrefixedDecideRule { + + private static final long serialVersionUID = 3L; + + { + setUseDomain(true); + } + public boolean getUseDomain() { + return (Boolean)kp.get("useDomain"); + } + public void setUseDomain(boolean useDomain) { + kp.put("useDomain", useDomain); + } + + private static final Logger logger = + Logger.getLogger(ScopePlusOneDecideRule.class.getName()); + + /** + * Constructor. + * @param name + */ + public ScopePlusOneDecideRule() { + super(); + } + + /** + * Evaluate whether given object comes from a URI which is in scope + * + * @param object to evaluate + * @return true if URI is either in scope or its via is + */ + @Override + protected DecideResult innerDecide(ProcessorURI uri) { + SurtPrefixSet set = getPrefixes(); + UURI u = uri.getUURI(); + // First, is the URI itself in scope? + boolean firstResult = isInScope(u, set); + if (logger.isLoggable(Level.FINE)) { + logger.fine("Tested scope of UURI itself '" + u + + " and result was " + firstResult); + } + if (firstResult == true) { + return DecideResult.ACCEPT; + } else { + // This object is not itself within scope, but + // see whether its via might be + UURI via = uri.getVia(); + if (via == null) { + // If there is no via and the URL doesn't match scope,reject it + return DecideResult.PASS; + } + // If the via is within scope, accept it + boolean result = isInScope (via, set); + if (logger.isLoggable(Level.FINE)) { + logger.fine("Tested via UURI '" + via + + " and result was " + result); + } + if (result) { + return DecideResult.ACCEPT; + } + } + return DecideResult.PASS; + } + + /** + * Synchronized get of prefix set to use. + * @param o Context object. + * + * @return SurtPrefixSet to use for check + * @see org.archive.modules.deciderules.surt.SurtPrefixedDecideRule#getPrefixes() + */ + protected synchronized SurtPrefixSet getPrefixes(/*StateProvider o*/) { + if (surtPrefixes == null) { + readPrefixes(); + } + return surtPrefixes; + } + + /** + * Patch the SURT prefix set so that it only includes the appropriate + * prefixes. + * @param o Context object. + * @see org.archive.modules.deciderules.surt.SurtPrefixedDecideRule#readPrefixes() + */ + @Override + protected void readPrefixes() { + buildSurtPrefixSet(); + // See whether Host or Domain was chosen + if (getUseDomain()) { + surtPrefixes.convertAllPrefixesToDomains(); + } else { + surtPrefixes.convertAllPrefixesToHosts(); + } + dumpSurtPrefixSet(); + } + + + //check that the URI is in scope + private boolean isInScope (Object o, SurtPrefixSet set) { + boolean iResult = false; + UURI u = (UURI)o; + if (u == null) { + return false; + } + String candidateSurt = u.getSurtForm(); + // also want to treat https as http + if (candidateSurt.startsWith("https:")) { + candidateSurt = "http:" + candidateSurt.substring(6); + } + if (set.containsPrefixOf(candidateSurt)){ + iResult = true; + } + return iResult; + } +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java new file mode 100644 index 00000000..247a4b57 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java @@ -0,0 +1,306 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.modules.deciderules.surt; + +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.Reader; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.archive.io.ReadSource; +import org.archive.modules.ProcessorURI; +import org.archive.modules.deciderules.PredicatedDecideRule; +import org.archive.modules.seeds.SeedListener; +import org.archive.modules.seeds.SeedModule; +import org.archive.net.UURI; +import org.archive.util.SurtPrefixSet; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.Lifecycle; + + + +/** + * Rule applies configured decision to any URIs that, when + * expressed in SURT form, begin with one of the prefixes + * in the configured set. + * + * The set can be filled with SURT prefixes implied or + * listed in the seeds file, or another external file. + * + * The "also-check-via" option to implement "one hop off" + * scoping derives from a contribution by Shifra Raffel + * of the California Digital Library. + * + * @author gojomo + */ +public class SurtPrefixedDecideRule extends PredicatedDecideRule + implements SeedListener, Lifecycle { + + private static final long serialVersionUID = 3L; + + private static final Logger logger = + Logger.getLogger(SurtPrefixedDecideRule.class.getName()); + + + /** + * Source file from which to infer SURT prefixes. Any URLs in file will be + * converted to the implied SURT prefix, and literal SURT prefixes may be + * listed on lines beginning with a '+' character. + */ + protected String surtsSourceFile = ""; + public String getSurtsSourceFile() { + return surtsSourceFile; + } + public void setSurtsSourceFile(String surtsSourceFile) { + this.surtsSourceFile = surtsSourceFile; + } + + /** + * Should seeds also be interpreted as SURT prefixes. + */ + protected boolean seedsAsSurtPrefixes = true; + public boolean getSeedsAsSurtPrefixes() { + return seedsAsSurtPrefixes; + } + public void setSeedsAsSurtPrefixes(boolean seedsAsSurtPrefixes) { + this.seedsAsSurtPrefixes = seedsAsSurtPrefixes; + } + + + /** + * Dump file to save SURT prefixes actually used: Useful debugging SURTs. + */ + protected String surtsDumpFile = ""; + public String getSurtsDumpFile() { + return surtsDumpFile; + } + public void setSurtsDumpFile(String surtsDumpFile) { + this.surtsDumpFile = surtsDumpFile; + } + + + /** + * Whether to rebuild the internal structures from source files (including + * seeds if appropriate) every time any configuration change occurs. If + * true, rule is rebuilt from sources even when (for example) unrelated new + * domain overrides are set. Rereading large source files can take a long + * time. + */ + protected boolean rebuildOnReconfig = true; + public boolean getRebuildOnReconfig() { + return rebuildOnReconfig; + } + public void setRebuildOnReconfig(boolean rebuildOnReconfig) { + this.rebuildOnReconfig = rebuildOnReconfig; + } + + /** + * Whether to also make the configured decision if a URI's 'via' URI (the + * URI from which it was discovered) in SURT form begins with any of the + * established prefixes. For example, can be used to ACCEPT URIs that are + * 'one hop off' URIs fitting the SURT prefixes. Default is false. + */ + { + setAlsoCheckVia(false); + } + public boolean getAlsoCheckVia() { + return (Boolean) kp.get("alsoCheckVia"); + } + public void setAlsoCheckVia(boolean checkVia) { + kp.put("alsoCheckVia", checkVia); + } + + protected SeedModule seeds; + public SeedModule getSeeds() { + return this.seeds; + } + @Autowired + public void setSeeds(SeedModule seeds) { + this.seeds = seeds; + } + + protected SurtPrefixSet surtPrefixes = null; + + /** + * Usual constructor. + */ + public SurtPrefixedDecideRule() { + } + + + public void start() { + if(isRunning()) { + return; + } + this.readPrefixes(); + } + + public boolean isRunning() { + return surtPrefixes != null; + } + + public void stop() { + surtPrefixes = null; + } + + /** + * Evaluate whether given object's URI is covered by the SURT prefix set + * + * @param object Item to evaluate. + * @return true if item, as SURT form URI, is prefixed by an item in the set + */ + @Override + protected boolean evaluate(ProcessorURI uri) { + if (getAlsoCheckVia()) { + if (innerDecide(uri.getVia())) { + return true; + } + } + + return innerDecide(uri.getUURI()); + } + + + private boolean innerDecide(UURI uuri) { + String candidateSurt; + candidateSurt = SurtPrefixSet.getCandidateSurt(uuri); + if (candidateSurt == null) { + return false; + } + if (getPrefixes().containsPrefixOf(candidateSurt)) { + return true; + } else { + return false; + } + } + + + /** + * Synchronized get of prefix set to use + * + * @return SurtPrefixSet to use for check + */ + private synchronized SurtPrefixSet getPrefixes() { + if (surtPrefixes == null) { + readPrefixes(); + } + return surtPrefixes; + } + + protected void readPrefixes() { + buildSurtPrefixSet(); + dumpSurtPrefixSet(); + } + + /** + * Dump the current prefixes in use to configured dump file (if any) + */ + protected void dumpSurtPrefixSet() { + // dump surts to file, if appropriate + String dumpPath = getSurtsDumpFile(); + if (!StringUtils.isEmpty(dumpPath)) { + File dump = new File(dumpPath); + try { + FileWriter fw = new FileWriter(dump); + try { + surtPrefixes.exportTo(fw); + } finally { + fw.close(); + } + } catch (IOException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } + } + + /** + * Construct the set of prefixes to use, from the seed list ( + * which may include both URIs and '+'-prefixed directives). + */ + protected void buildSurtPrefixSet() { + SurtPrefixSet newSurtPrefixes = new SurtPrefixSet(); + Reader fr = null; + + // read SURTs from file, if appropriate + String sourcePath = getSurtsSourceFile(); + if (!StringUtils.isEmpty(sourcePath)) { + File source = new File(sourcePath); + try { + fr = new FileReader(source); + try { + newSurtPrefixes.importFromMixed(fr, true); + } finally { + fr.close(); + } + } catch (IOException e) { + logger.log(Level.SEVERE,"Problem reading SURTs source file: "+e,e); + // continue: operator will see severe log message or alert + } + } + + // interpret seeds as surts, if appropriate + boolean deduceFromSeeds = getSeedsAsSurtPrefixes(); + if(deduceFromSeeds) { + if(seeds instanceof ReadSource) { + // scan text + fr = ((ReadSource)seeds).getReader(); + newSurtPrefixes.importFromMixed(fr, deduceFromSeeds); + IOUtils.closeQuietly(fr); + } else { + // just deduce from URIs + for(UURI u : seeds) { + newSurtPrefixes.addFromPlain(u.toCustomString()); + } + } + } + surtPrefixes = newSurtPrefixes; + } + + /** + * Re-read prefixes after an update. + */ + public synchronized void noteReconfiguration(/*KeyChangeEvent event*/) { + if (getRebuildOnReconfig()) { + readPrefixes(); + } + // TODO: make conditional on file having actually changed, + // perhaps by remembering mod-time + } + + public synchronized void addedSeed(final ProcessorURI curi) { + SurtPrefixSet newSurtPrefixes = (SurtPrefixSet) surtPrefixes.clone(); + newSurtPrefixes.add(prefixFrom(curi.toString())); + surtPrefixes = newSurtPrefixes; + } + + public void seedsRefreshed() { + // TODO update? + } + + protected String prefixFrom(String uri) { + return SurtPrefixSet.prefixFromPlainForceHttp(uri); + } +}//EOC diff --git a/modules/src/main/java/org/archive/modules/extractor/AggressiveExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/AggressiveExtractorHTML.java new file mode 100644 index 00000000..4744a316 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/extractor/AggressiveExtractorHTML.java @@ -0,0 +1,72 @@ +/* + * AggressiveExtractorHTML + * + * $Id$ + * + * Created on Jan 6, 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +package org.archive.modules.extractor; + +import java.util.logging.Logger; + +import org.archive.modules.ProcessorURI; + +/** + * Extended version of ExtractorHTML with more aggressive javascript link + * extraction where javascript code is parsed first with general HTML tags + * regexp, and than by javascript speculative link regexp. + * + * @author Igor Ranitovic + * + */ +public class AggressiveExtractorHTML +extends ExtractorHTML { + private static final long serialVersionUID = 3L; + + static Logger logger = + Logger.getLogger(AggressiveExtractorHTML.class.getName()); + + public AggressiveExtractorHTML() { + } + + protected void processScript(ProcessorURI curi, CharSequence sequence, + int endOfOpenTag) { + super.processScript(curi, sequence, endOfOpenTag); + // then, proccess entire javascript code as html code + // this may cause a lot of false positves + processGeneralTag(curi, sequence.subSequence(0,6), + sequence.subSequence(endOfOpenTag, sequence.length())); + } + + /* (non-Javadoc) + * @see org.archive.crawler.framework.Processor#report() + */ + public String report() { + StringBuffer ret = new StringBuffer(256); + ret.append("Processor: org.archive.crawler.extractor.ExtractorHTML2\n"); + ret.append(" Function: Link extraction on HTML documents " + + "(including embedded CSS)\n"); + ret.append(" ProcessorURRIs handled: " + numberOfCURIsHandled + "\n"); + ret.append(" Links extracted: " + numberOfLinksExtracted + "\n\n"); + return ret.toString(); + } +} diff --git a/modules/src/main/java/org/archive/modules/extractor/ContentExtractor.java b/modules/src/main/java/org/archive/modules/extractor/ContentExtractor.java new file mode 100644 index 00000000..2de29a79 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/extractor/ContentExtractor.java @@ -0,0 +1,117 @@ +/* Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * ContentExtractor.java + * Created on October 5, 2006 + * + * $Header$ + */ +package org.archive.modules.extractor; + +import org.archive.modules.ProcessorURI; + + +/** + * Extracts link from the fetched content of a URI, as opposed to its headers. + * + * @author pjack + */ +public abstract class ContentExtractor extends Extractor { + + + /** + * Extracts links + */ + final protected void extract(ProcessorURI uri) { + boolean finished = innerExtract(uri); + if (finished) { + uri.linkExtractorFinished(); + } + } + + + /** + * Determines if links should be extracted from the given URI. This + * method performs three checks. The first is to check the URI's + * {@link ExtractorURI#hasBeenLinkExtracted()} result. If that + * result is true, then this method returns false, as some other + * extractor has claimed that links are already extracted. + * + *

      Next, this method checks that the content length of the URI is + * greater than zero (in other words, that there is actually content + * for links to be extracted from). If the content length of the URI + * is zero or less, then this method returns false. + * + *

      Finally, this method delegates to {@link #innerExtract(ExtractorURI)} + * and returns that result. + * + * @param uri the URI to check + * @return true if links should be extracted from the URI, + * false otherwise + */ + final protected boolean shouldProcess(ProcessorURI uri) { + if (uri.hasBeenLinkExtracted()) { + return false; + } + if (uri.getContentLength() <= 0) { + return false; + } + if (!shouldExtract(uri)) { + return false; + } + return true; + } + + + /** + * Determines if otherwise valid URIs should have links extracted or not. + * The given URI will not have its + * {@link ExtractorURI#hasBeenLinkExtracted()} flag set, and its + * content length will be greater than zero. Subclasses should + * implement this method to perform additional checks. For instance, + * the {@link ExtractorHTML} implementation checks that the content-type + * of the given URI is text/html. + * + * @param uri the URI to check + * @return true if links should be extracted from that URI, false + * otherwise + */ + protected abstract boolean shouldExtract(ProcessorURI uri); + + + /** + * Actually extracts links. The given URI will have passed the three + * checks described in {@link #shouldProcess(ExtractorURI)}. Subclasses + * should implement this method to discover outlinks in the URI's + * content stream. For instance, {@link ExtractorHTML} extracts links + * from Anchor tags and so on. + * + *

      This method should only return true if extraction completed + * successfully. If not (for instance, if an IO error occurred), then + * this method should return false. Returning false indicates to the + * pipeline that downstream extractors should attempt to extract links + * themselves. Returning true indicates that downstream extractors + * should be skipped. + * + * @param uri the URI whose links to extract + * @return true if link extraction finished; false if downstream + * extractors should attempt to extract links + */ + protected abstract boolean innerExtract(ProcessorURI uri); + +} diff --git a/modules/src/main/java/org/archive/modules/extractor/ContentExtractorTestBase.java b/modules/src/main/java/org/archive/modules/extractor/ContentExtractorTestBase.java new file mode 100644 index 00000000..dff26cf2 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/extractor/ContentExtractorTestBase.java @@ -0,0 +1,147 @@ +/* Copyright (C) 2006 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * ContentExtractorTest.java + * Created on October 5, 2006 + * + * $Header$ + */ +package org.archive.modules.extractor; + + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.InputStream; +import java.util.Collections; + +import org.archive.modules.DefaultProcessorURI; +import org.archive.modules.ProcessorTestBase; +import org.archive.net.UURI; +import org.archive.net.UURIFactory; +import org.archive.state.ModuleTestBase; +import org.archive.util.Recorder; + + +/** + * Abstract base class for unit testing ContentExtractor implementations. + * + * @author pjack + */ +public abstract class ContentExtractorTestBase extends ProcessorTestBase { + + + /** + * An extractor created during the setUp. + */ + protected Extractor extractor; + + + /** + * Sets up the {@link #extractor} and + * {@link ModuleTestBase#processorClass} + * fields. + */ + final public void setUp() { + extractor = makeExtractor(); + } + + + @Override + protected Object makeModule() { + return makeExtractor(); + } + + + /** + * Subclasses should return an Extractor instance to test. + * + * @return an Extractor instance to test + */ + protected abstract Extractor makeExtractor(); + + + /** + * Returns a DefaultProcessorURI for testing purposes. + * + * @return a DefaultProcessorURI + * @throws Exception just in case + */ + protected DefaultProcessorURI defaultURI() throws Exception { + UURI uuri = UURIFactory.getInstance("http://www.archive.org/start/"); + return new DefaultProcessorURI(uuri, LinkContext.NAVLINK_MISC); + } + + + /** + * Tests that a URI with a zero content length has no links extracted. + * + * @throws Exception just in case + */ + public void testZeroContent() throws Exception { + DefaultProcessorURI uri = defaultURI(); + Recorder recorder = createRecorder(""); + uri.setContentType("text/plain"); + uri.setRecorder(recorder); + extractor.process(uri); + assertEquals(0, uri.getOutLinks().size()); + assertNoSideEffects(uri); + } + + + /** + * Tests that a URI whose linkExtractionFinished flag has been set has + * no links extracted. + * + * @throws Exception just in case + */ + public void testFinished() throws Exception { + DefaultProcessorURI uri = defaultURI(); + uri.linkExtractorFinished(); + extractor.process(uri); + assertEquals(0, uri.getOutLinks().size()); + assertNoSideEffects(uri); + } + + + /** + * Asserts that the given URI has no URI errors, no localized errors, and + * no annotations. + * + * @param uri the URI to test + */ + protected static void assertNoSideEffects(DefaultProcessorURI uri) { + assertEquals(0, uri.getUriErrors().size()); + assertEquals(0, uri.getNonFatalFailures().size()); + assertEquals(Collections.EMPTY_LIST, uri.getAnnotations()); + } + + + + public static Recorder createRecorder(String content) + throws Exception { + File temp = File.createTempFile("test", ".tmp"); + Recorder recorder = new Recorder(temp, 1024, 1024); + byte[] b = content.getBytes(); // FIXME: Allow other encodings? + ByteArrayInputStream bais = new ByteArrayInputStream(b); + InputStream is = recorder.inputWrap(bais); + for (int x = is.read(); x >= 0; x = is.read()); + is.close(); + return recorder; + } + +} diff --git a/modules/src/main/java/org/archive/modules/extractor/CustomSWFTags.java b/modules/src/main/java/org/archive/modules/extractor/CustomSWFTags.java new file mode 100644 index 00000000..82cf9ef6 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/extractor/CustomSWFTags.java @@ -0,0 +1,58 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.modules.extractor; + +import java.io.IOException; +import java.util.Vector; + +import com.anotherbigidea.flash.interfaces.SWFActions; +import com.anotherbigidea.flash.writers.SWFTagTypesImpl; + +/** + * Overwrite action tags, that may hold URI, to use CrawlUriSWFAction + * action. + * + * @author Igor Ranitovic + */ +@SuppressWarnings("unchecked") +public class CustomSWFTags extends SWFTagTypesImpl { + SWFActions actions; + + public CustomSWFTags(SWFActions a) { + super(null); + actions = a; + } + + public SWFActions tagDefineButton(int id, Vector buttonRecords) + throws IOException { + + return actions; + } + + public SWFActions tagDefineButton2(int id, boolean trackAsMenu, + Vector buttonRecord2s) throws IOException { + + return actions; + } + + public SWFActions tagDoAction() throws IOException { + return actions; + } +} diff --git a/modules/src/main/java/org/archive/modules/extractor/Extractor.java b/modules/src/main/java/org/archive/modules/extractor/Extractor.java new file mode 100644 index 00000000..1a1eed4e --- /dev/null +++ b/modules/src/main/java/org/archive/modules/extractor/Extractor.java @@ -0,0 +1,131 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.modules.extractor; + + +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.httpclient.URIException; +import org.archive.modules.Processor; +import org.archive.modules.ProcessorURI; +import org.archive.net.UURI; +import org.archive.net.UURIFactory; +import org.springframework.beans.factory.annotation.Autowired; + + +/** + * Extracts links from fetched URIs. This class provides error handling + * for some common issues that occur when parsing document content. You + * almost certainly want to subclass {@link ContentExtractor} instead of + * this class. + * + * @author pjack + */ +public abstract class Extractor extends Processor { + + + /** Logger. */ + private static final Logger logger = + Logger.getLogger(Extractor.class.getName()); + + public static final ExtractorParameters DEFAULT_PARAMETERS = + new ExtractorParameters() { + public int getMaxOutlinks() { + return 6000; + } + }; + + transient protected UriErrorLoggerModule loggerModule; + public UriErrorLoggerModule getLoggerModule() { + return this.loggerModule; + } + @Autowired + public void setLoggerModule(UriErrorLoggerModule loggerModule) { + this.loggerModule = loggerModule; + } + + transient ExtractorParameters extractorParameters; + public ExtractorParameters getExtractorParameters() { + return extractorParameters; + } + @Autowired + public void setExtractorParameters(ExtractorParameters helper) { + this.extractorParameters = helper; + } + + /** + * Processes the given URI. This method just delegates to + * {@link #extract(ExtractorURI)}, catching runtime exceptions and + * errors that are usually non-fatal, to highlight them in the + * relevant log(s). + * + *

      Notably, StackOverflowError is caught here, as that seems to + * happen a lot when dealing with document parsing APIs. + * + * @param uri the URI to extract links from + */ + final protected void innerProcess(ProcessorURI uri) + throws InterruptedException { + try { + extract(uri); + } catch (NullPointerException npe) { + handleException(uri, npe); + } catch (StackOverflowError soe) { + handleException(uri, soe); + } catch (java.nio.charset.CoderMalfunctionError cme) { + // See http://sourceforge.net/tracker/index.php?func=detail&aid=1540222&group_id=73833&atid=539099 + handleException(uri, cme); + } + } + + + private void handleException(ProcessorURI uri, Throwable t) { + // both annotate (to highlight in crawl log) & add as local-error + uri.getAnnotations().add("err=" + t.getClass().getName()); + uri.getNonFatalFailures().add(t); + // also log as INFO + // TODO: remove as redundant, given nonfatal logging? + logger.log(Level.INFO, "Exception", t); + } + + + /** + * Extracts links from the given URI. Subclasses should use + * {@link ExtractorURI#getInputStream()} or + * {@link ExtractorURI#getCharSequence()} to process the content of the + * URI. Any links that are discovered should be added to the + * {@link ExtractorURI#getOutLinks()} set. + * + * @param uri the uri to extract links from + */ + protected abstract void extract(ProcessorURI uri); + + + public void logUriError(URIException e, UURI uuri, + CharSequence l) { + if (e.getReasonCode() == UURIFactory.IGNORED_SCHEME) { + // don't log those that are intentionally ignored + return; + } + loggerModule.logUriError(e, uuri, l); + } + +} diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java new file mode 100644 index 00000000..70921d42 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java @@ -0,0 +1,179 @@ +/* + * ExtractorCSS + * + * $Id$ + * + * Created on Jan 6, 2004 + * + * Copyright (C) 2004 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.modules.extractor; + +import java.io.IOException; +import java.util.logging.Logger; +import java.util.regex.Matcher; + +import org.apache.commons.httpclient.URIException; +import org.archive.io.ReplayCharSequence; +import org.archive.modules.ProcessorURI; +import org.archive.net.UURI; +import org.archive.util.DevUtils; +import org.archive.util.TextUtils; + +/** + * This extractor is parsing URIs from CSS type files. + * The format of a CSS URL value is 'url(' followed by optional white space + * followed by an optional single quote (') or double quote (") character + * followed by the URL itself followed by an optional single quote (') or + * double quote (") character followed by optional white space followed by ')'. + * Parentheses, commas, white space characters, single quotes (') and double + * quotes (") appearing in a URL must be escaped with a backslash: + * '\(', '\)', '\,'. Partial URLs are interpreted relative to the source of + * the style sheet, not relative to the document. + * Source: www.w3.org + * + * @author Igor Ranitovic + * + **/ + +public class ExtractorCSS extends ContentExtractor { + + + private static final long serialVersionUID = 2L; + + private static Logger logger = + Logger.getLogger("org.archive.crawler.extractor.ExtractorCSS"); + + private static String ESCAPED_AMP = "&"; + // CSS escapes: "Parentheses, commas, whitespace characters, single + // quotes (') and double quotes (") appearing in a URL must be + // escaped with a backslash" + static final String CSS_BACKSLASH_ESCAPE = "\\\\([,'\"\\(\\)\\s])"; + + /** + * CSS URL extractor pattern. + * + * This pattern extracts URIs for CSS files + **/ +// static final String CSS_URI_EXTRACTOR = +// "url[(]\\s*([\"\']?)([^\\\"\\'].*?)\\1\\s*[)]"; + static final String CSS_URI_EXTRACTOR = + "(?i)(?:@import (?:url[(]|)|url[(])\\s*([\\\"\']?)" + // G1 + "([^\\\"\'].{0,"+UURI.MAX_URL_LENGTH+"}?)\\1\\s*[);]"; // G2 + // GROUPS: + // (G1) optional ' or " + // (G2) URI + + private long numberOfLinksExtracted = 0; + + /** + */ + public ExtractorCSS() { + } + + + @Override + protected boolean shouldExtract(ProcessorURI curi) { + String mimeType = curi.getContentType(); + if (mimeType == null) { + return false; // FIXME: This check should be unnecessary + } + if ((mimeType.toLowerCase().indexOf("css") < 0) && + (!curi.toString().toLowerCase().endsWith(".css"))) { + return false; + } + return true; + } + + /** + * @param curi Crawl URI to process. + */ + public boolean innerExtract(ProcessorURI curi) { + ReplayCharSequence cs = null; + try { + cs = curi.getRecorder().getReplayCharSequence(); + } catch (IOException e) { + logger.severe("Failed getting ReplayCharSequence: " + e.getMessage()); + } + if (cs == null) { + logger.warning("Failed getting ReplayCharSequence: " + + curi.toString()); + return false; + } + + // We have a ReplayCharSequence open. Wrap all in finally so we + // for sure close it before we leave. + try { + this.numberOfLinksExtracted += + processStyleCode(this, curi, cs); + // Set flag to indicate that link extraction is completed. + return true; + } finally { + if (cs != null) { + try { + cs.close(); + } catch (IOException ioe) { + logger.warning(TextUtils.exceptionToString( + "Failed close of ReplayCharSequence.", ioe)); + } + } + } + } + + public static long processStyleCode(Extractor ext, + ProcessorURI curi, CharSequence cs) { + long foundLinks = 0; + Matcher uris = null; + String cssUri; + try { + uris = TextUtils.getMatcher(CSS_URI_EXTRACTOR, cs); + while (uris.find()) { + cssUri = uris.group(2); + // TODO: Escape more HTML Entities. + cssUri = TextUtils.replaceAll(ESCAPED_AMP, cssUri, "&"); + // Remove backslashes when used as escape character in CSS URL + cssUri = TextUtils.replaceAll(CSS_BACKSLASH_ESCAPE, cssUri, + "$1"); + foundLinks++; + int max = ext.getExtractorParameters().getMaxOutlinks(); + try { + Link.addRelativeToBase(curi, max, cssUri, + LinkContext.EMBED_MISC, Hop.EMBED); + } catch (URIException e) { + ext.logUriError(e, curi.getUURI(), cssUri); + } + } + } catch (StackOverflowError e) { + DevUtils.warnHandle(e, "ExtractorCSS StackOverflowError"); + } finally { + TextUtils.recycleMatcher(uris); + } + return foundLinks; + } + + public String report() { + StringBuffer ret = new StringBuffer(); + ret.append("Processor: org.archive.crawler.extractor.ExtractorCSS\n"); + ret.append(" Function: Link extraction on Cascading Style Sheets (.css)\n"); + ret.append(" ExtractorURIs handled: " + getURICount() + "\n"); + ret.append(" Links extracted: " + numberOfLinksExtracted + "\n\n"); + + return ret.toString(); + } +} diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java new file mode 100644 index 00000000..374f10a1 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java @@ -0,0 +1,141 @@ +/* Copyright (C) 2003 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Created on Jul 7, 2003 + * + */ +package org.archive.modules.extractor; + +import java.io.IOException; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.httpclient.URIException; +import org.archive.io.ReplayInputStream; +import org.archive.io.SeekReader; +import org.archive.io.SeekReaderCharSequence; +import org.archive.modules.ProcessorURI; +import org.archive.net.UURI; +import org.archive.net.UURIFactory; +import org.archive.util.ms.Doc; + +/** + * This class allows the caller to extract href style links from word97-format word documents. + * + * @author Parker Thompson + * + */ +public class ExtractorDOC extends ContentExtractor { + + private static final long serialVersionUID = 3L; + + private static Pattern PATTERN = Pattern.compile("HYPERLINK.*?\"(.*?)\""); + + private static Logger logger = + Logger.getLogger("org.archive.crawler.extractor.ExtractorDOC"); + private long numberOfCURIsHandled = 0; + private long numberOfLinksExtracted = 0; + + /** + * @param name + */ + public ExtractorDOC() { + } + + + @Override + protected boolean shouldExtract(ProcessorURI uri) { + String mimeType = uri.getContentType(); + if (mimeType == null) { + return false; + } + return mimeType.toLowerCase().startsWith("application/msword"); + } + + + /** + * Processes a word document and extracts any hyperlinks from it. + * This only extracts href style links, and does not examine the actual + * text for valid URIs. + * @param curi ProcessorURI to process. + */ + protected boolean innerExtract(ProcessorURI curi){ + int links = 0; + ReplayInputStream documentStream = null; + SeekReader docReader = null; + + // Get the doc as a repositionable reader + try + { + documentStream = curi.getRecorder().getRecordedInput(). + getContentReplayInputStream(); + + if (documentStream==null) { + // TODO: note problem + return false; + } + + docReader = Doc.getText(documentStream); + } catch(Exception e){ + curi.getNonFatalFailures().add(e); + return false; + } finally { + try { + documentStream.close(); + } catch (IOException ignored) { + + } + } + + CharSequence cs = new SeekReaderCharSequence(docReader, 0); + Matcher m = PATTERN.matcher(cs); + while (m.find()) { + links++; + addLink(curi, m.group(1)); + } + logger.fine(curi + " has " + links + " links."); + return true; + } + + + private void addLink(ProcessorURI curi, String hyperlink) { + try { + UURI dest = UURIFactory.getInstance(curi.getUURI(), hyperlink); + LinkContext lc = LinkContext.NAVLINK_MISC; + Link link = new Link(curi.getUURI(), dest, lc, Hop.NAVLINK); + curi.getOutLinks().add(link); + } catch (URIException e1) { + logUriError(e1, curi.getUURI(), hyperlink); + } + numberOfLinksExtracted++; + } + + /* (non-Javadoc) + * @see org.archive.crawler.framework.Processor#report() + */ + public String report() { + StringBuffer ret = new StringBuffer(); + ret.append("Processor: org.archive.crawler.extractor.ExtractorDOC\n"); + ret.append(" Function: Link extraction on MS Word documents (.doc)\n"); + ret.append(" ProcessorURIs handled: " + numberOfCURIsHandled + "\n"); + ret.append(" Links extracted: " + numberOfLinksExtracted + "\n\n"); + + return ret.toString(); + } +} diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java new file mode 100644 index 00000000..5a4963ed --- /dev/null +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -0,0 +1,865 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.modules.extractor; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.httpclient.URIException; +import org.archive.io.ReplayCharSequence; +import org.archive.modules.ProcessorURI; +import org.archive.modules.net.RobotsHonoringPolicy; +import org.archive.net.UURI; +import org.archive.net.UURIFactory; +import org.springframework.beans.factory.InitializingBean; +import org.archive.util.DevUtils; +import org.archive.util.TextUtils; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Basic link-extraction, from an HTML content-body, + * using regular expressions. + * + * @author gojomo + * + */ +public class ExtractorHTML extends ContentExtractor implements InitializingBean { + + private static final long serialVersionUID = 2L; + + private static Logger logger = + Logger.getLogger(ExtractorHTML.class.getName()); + + + + private final static String MAX_ELEMENT_REPLACE = "MAX_ELEMENT"; + + private final static String MAX_ATTR_NAME_REPLACE = "MAX_ATTR_NAME"; + + private final static String MAX_ATTR_VAL_REPLACE = "MAX_ATTR_VAL"; + + public final static String A_META_ROBOTS = "meta-robots"; + + + /** + * Compiled relevant tag extractor. + * + *

      + * This pattern extracts either: + *

    2. (1) whole <script>...</script> or + *
    3. (2) <style>...</style> or + *
    4. (3) <meta ...> or + *
    5. (4) any other open-tag with at least one attribute + * (eg matches "<a href='boo'>" but not "</a>" or "<br>") + *

      + * groups: + *

    6. 1: SCRIPT SRC=foo>boo</SCRIPT + *
    7. 2: just script open tag + *
    8. 3: STYLE TYPE=moo>zoo</STYLE + *
    9. 4: just style open tag + *
    10. 5: entire other tag, without '<' '>' + *
    11. 6: element + *
    12. 7: META + *
    13. 8: !-- comment -- + */ +// version w/ less unnecessary backtracking + + { + setMaxElementLength(1024); // no limit + } + public int getMaxElementLength() { + return (Integer) kp.get("maxElementLength"); + } + public void setMaxElementLength(int max) { + kp.put("maxElementLength",max); + } + + static final String RELEVANT_TAG_EXTRACTOR = + "(?is)<(?:((script[^>]*+)>.*?]*+)>.*?]*+)" + // 5, 6, 7 + "|(!--.*?--))>"; // 8 + +// version w/ problems with unclosed script tags +// static final String RELEVANT_TAG_EXTRACTOR = +// "(?is)<(?:((script.*?)>.*?.*?"; + + + +// // this pattern extracts 'href' or 'src' attributes from +// // any open-tag innards matched by the above +// static Pattern RELEVANT_ATTRIBUTE_EXTRACTOR = Pattern.compile( +// "(?is)(\\w+)(?:\\s+|(?:\\s.*?\\s))(?:(href)|(src))\\s*=(?:(?:\\s*\"(.+?)\")|(?:\\s*'(.+?)')|(\\S+))"); +// +// // this pattern extracts 'robots' attributes +// static Pattern ROBOTS_ATTRIBUTE_EXTRACTOR = Pattern.compile( +// "(?is)(\\w+)\\s+.*?(?:(robots))\\s*=(?:(?:\\s*\"(.+)\")|(?:\\s*'(.+)')|(\\S+))"); + + { + setMaxAttrNameLength(1024); // 1K + } + + public int getMaxAttrNameLength() { + return (Integer) kp.get("maxAttrNameLength"); + } + + public void setMaxAttrNameLength(int max) { + kp.put("maxAttrNameLength", max); + } + + + { + setMaxAttrValLength(16384); // 16K + } + + public int getMaxAttrValLength() { + return (Integer) kp.get("maxAttrValLength"); + } + + public void setMaxAttrValLength(int max) { + kp.put("maxAttrValLength", max); + } + + // TODO: perhaps cut to near MAX_URI_LENGTH + + // this pattern extracts attributes from any open-tag innards + // matched by the above. attributes known to be URIs of various + // sorts are matched specially + static final String EACH_ATTRIBUTE_EXTRACTOR = + "(?is)\\s?((href)|(action)|(on\\w*)" // 1, 2, 3, 4 + +"|((?:src)|(?:lowsrc)|(?:background)|(?:cite)|(?:longdesc)" // ... + +"|(?:usemap)|(?:profile)|(?:datasrc))" // 5 + +"|(codebase)|((?:classid)|(?:data))|(archive)|(code)" // 6, 7, 8, 9 + +"|(value)|(style)|(method)" // 10, 11, 12 + +"|([-\\w]{1,"+MAX_ATTR_NAME_REPLACE+"}))" // 13 + +"\\s*=\\s*" + +"(?:(?:\"(.{0,"+MAX_ATTR_VAL_REPLACE+"}?)(?:\"|$))" // 14 + +"|(?:'(.{0,"+MAX_ATTR_VAL_REPLACE+"}?)(?:'|$))" // 15 + +"|(\\S{1,"+MAX_ATTR_VAL_REPLACE+"}))"; // 16 + // groups: + // 1: attribute name + // 2: HREF - single URI relative to doc base, or occasionally javascript: + // 3: ACTION - single URI relative to doc base, or occasionally javascript: + // 4: ON[WHATEVER] - script handler + // 5: SRC,LOWSRC,BACKGROUND,CITE,LONGDESC,USEMAP,PROFILE, or DATASRC + // single URI relative to doc base + // 6: CODEBASE - a single URI relative to doc base, affecting other + // attributes + // 7: CLASSID, DATA - a single URI relative to CODEBASE (if supplied) + // 8: ARCHIVE - one or more space-delimited URIs relative to CODEBASE + // (if supplied) + // 9: CODE - a single URI relative to the CODEBASE (is specified). + // 10: VALUE - often includes a uri path on forms + // 11: STYLE - inline attribute style info + // 12: METHOD - form GET/POST + // 13: any other attribute + // 14: double-quote delimited attr value + // 15: single-quote delimited attr value + // 16: space-delimited attr value + + + // much like the javascript likely-URI extractor, but + // without requiring quotes -- this can indicate whether + // an HTML tag attribute that isn't definitionally a + // URI might be one anyway, as in form-tag VALUE attributes + static final String LIKELY_URI_PATH = + "(\\.{0,2}[^\\.\\n\\r\\s\"']*(\\.[^\\.\\n\\r\\s\"']+)+)"; + static final String WHITESPACE = "\\s"; + static final String CLASSEXT =".class"; + static final String APPLET = "applet"; + static final String BASE = "base"; + static final String LINK = "link"; + static final String FRAME = "frame"; + static final String IFRAME = "iframe"; + + + /** + * If true, FRAME/IFRAME SRC-links are treated as embedded resources (like + * IMG, 'E' hop-type), otherwise they are treated as navigational links. + * Default is true. + */ + { + setTreatFramesAsEmbedLinks(true); + } + public boolean getTreatFramesAsEmbedLinks() { + return (Boolean) kp.get("treatFramesAsEmbedLinks"); + } + public void setTreatFramesAsEmbedLinks(boolean asEmbeds) { + kp.put("treatFramesAsEmbedLinks",asEmbeds); + } + + /** + * If true, URIs appearing as the ACTION attribute in HTML FORMs are + * ignored. Default is false. + */ + { + setIgnoreFormActionUrls(false); + } + public boolean getIgnoreFormActionUrls() { + return (Boolean) kp.get("ignoreFormActionUrls"); + } + public void setIgnoreFormActionUrls(boolean ignoreActions) { + kp.put("ignoreFormActionUrls",ignoreActions); + } + + /** + * If true, only ACTION URIs with a METHOD of GET (explicit or implied) + * are extracted. Default is true. + */ + { + setExtractOnlyFormGets(true); + } + public boolean getExtractOnlyFormGets() { + return (Boolean) kp.get("extractOnlyFormGets"); + } + public void setExtractOnlyFormGets(boolean onlyGets) { + kp.put("extractOnlyFormGets",onlyGets); + } + + /** + * If true, in-page Javascript is scanned for strings that + * appear likely to be URIs. This typically finds both valid + * and invalid URIs, and attempts to fetch the invalid URIs + * sometimes generates webmaster concerns over odd crawler + * behavior. Default is true. + */ + { + setExtractJavascript(true); + } + public boolean getExtractJavascript() { + return (Boolean) kp.get("extractJavascript"); + } + public void setExtractJavascript(boolean extractJavascript) { + kp.put("extractJavascript",extractJavascript); + } + + /** + * If true, strings that look like URIs found in unusual places (such as + * form VALUE attributes) will be extracted. This typically finds both valid + * and invalid URIs, and attempts to fetch the invalid URIs sometimes + * generate webmaster concerns over odd crawler behavior. Default is true. + */ + { + setExtractValueAttributes(true); + } + public boolean getExtractValueAttributes() { + return (Boolean) kp.get("extractValueAttributes"); + } + public void setExtractValueAttributes(boolean extractValueAttributes) { + kp.put("extractValueAttributes",extractValueAttributes); + } + + /** + * If true, URIs which end in typical non-HTML extensions (such as .gif) + * will not be scanned as if it were HTML. Default is true. + */ + { + setIgnoreUnexpectedHtml(true); + } + public boolean getIgnoreUnexpectedHtml() { + return (Boolean) kp.get("ignoreUnexpectedHtml"); + } + public void setIgnoreUnexpectedHtml(boolean ignoreUnexpectedHtml) { + kp.put("ignoreUnexpectedHtml",ignoreUnexpectedHtml); + } + + /** + * The robots honoring policy to use when considering a robots META tag. + */ + public RobotsHonoringPolicy getRobotsHonoringPolicy() { + return (RobotsHonoringPolicy) kp.get("robotsHonoringPolicy"); + } + @Autowired + public void setRobotsHonoringPolicy(RobotsHonoringPolicy policy) { + kp.put("robotsHonoringPolicy",policy); + } + + protected long numberOfCURIsHandled = 0; + protected long numberOfLinksExtracted = 0; + + + RobotsHonoringPolicy honoringPolicy; + + private Pattern relevantTagExtractor; + private Pattern eachAttributeExtractor; + + + public ExtractorHTML() { + } + + public void afterPropertiesSet() { + String regex = RELEVANT_TAG_EXTRACTOR; + regex = regex.replace(MAX_ELEMENT_REPLACE, + Integer.toString(getMaxElementLength())); + this.relevantTagExtractor = Pattern.compile(regex); + + regex = EACH_ATTRIBUTE_EXTRACTOR; + regex = regex.replace(MAX_ATTR_NAME_REPLACE, + Integer.toString(getMaxAttrNameLength())); + regex = regex.replace(MAX_ATTR_VAL_REPLACE, + Integer.toString(getMaxAttrValLength())); + this.eachAttributeExtractor = Pattern.compile(regex); + } + + + protected void processGeneralTag(ProcessorURI curi, CharSequence element, + CharSequence cs) { + + Matcher attr = eachAttributeExtractor.matcher(cs); + + // Just in case it's an OBJECT or APPLET tag + String codebase = null; + ArrayList resources = null; + + // Just in case it's a FORM + CharSequence action = null; + CharSequence actionContext = null; + CharSequence method = null; + + final boolean framesAsEmbeds = + getTreatFramesAsEmbedLinks(); + + final boolean ignoreFormActions = + getIgnoreFormActionUrls(); + + final boolean extractValueAttributes = + getExtractValueAttributes(); + + final String elementStr = element.toString(); + + while (attr.find()) { + int valueGroup = + (attr.start(14) > -1) ? 14 : (attr.start(15) > -1) ? 15 : 16; + int start = attr.start(valueGroup); + int end = attr.end(valueGroup); + assert start >= 0: "Start is: " + start + ", " + curi; + assert end >= 0: "End is :" + end + ", " + curi; + CharSequence value = cs.subSequence(start, end); + value = TextUtils.unescapeHtml(value); + if (attr.start(2) > -1) { + // HREF + CharSequence context = elementContext(element, attr.group(2)); + if(elementStr.equalsIgnoreCase(LINK)) { + // elements treated as embeds (css, ico, etc) + processEmbed(curi, value, context); + } else { + // other HREFs treated as links + processLink(curi, value, context); + } + if (elementStr.equalsIgnoreCase(BASE)) { + try { + UURI base = UURIFactory.getInstance(value.toString()); + curi.setBaseURI(base); + } catch (URIException e) { + logUriError(e, curi.getUURI(), value); + } + } + } else if (attr.start(3) > -1) { + // ACTION + if (!ignoreFormActions) { + action = value; + actionContext = elementContext(element, attr.group(3)); + // handling finished only at end (after METHOD also collected) + } + } else if (attr.start(4) > -1) { + // ON____ + processScriptCode(curi, value); // TODO: context? + } else if (attr.start(5) > -1) { + // SRC etc. + CharSequence context = elementContext(element, attr.group(5)); + + // true, if we expect another HTML page instead of an image etc. + final Hop hop; + + if(!framesAsEmbeds + && (elementStr.equalsIgnoreCase(FRAME) || elementStr + .equalsIgnoreCase(IFRAME))) { + hop = Hop.NAVLINK; + } else { + hop = Hop.EMBED; + } + processEmbed(curi, value, context, hop); + } else if (attr.start(6) > -1) { + // CODEBASE + codebase = (value instanceof String)? + (String)value: value.toString(); + CharSequence context = elementContext(element, + attr.group(6)); + processEmbed(curi, codebase, context); + } else if (attr.start(7) > -1) { + // CLASSID, DATA + if (resources == null) { + resources = new ArrayList(); + } + resources.add(value.toString()); + } else if (attr.start(8) > -1) { + // ARCHIVE + if (resources==null) { + resources = new ArrayList(); + } + String[] multi = TextUtils.split(WHITESPACE, value); + for(int i = 0; i < multi.length; i++ ) { + resources.add(multi[i]); + } + } else if (attr.start(9) > -1) { + // CODE + if (resources==null) { + resources = new ArrayList(); + } + // If element is applet and code value does not end with + // '.class' then append '.class' to the code value. + if (elementStr.equalsIgnoreCase(APPLET) && + !value.toString().toLowerCase().endsWith(CLASSEXT)) { + resources.add(value.toString() + CLASSEXT); + } else { + resources.add(value.toString()); + } + } else if (attr.start(10) > -1) { + // VALUE, with possibility of URI + if (extractValueAttributes + && TextUtils.matches(LIKELY_URI_PATH, value)) { + CharSequence context = elementContext(element, + attr.group(10)); + processLink(curi,value, context); + } + + } else if (attr.start(11) > -1) { + // STYLE inline attribute + // then, parse for URIs + this.numberOfLinksExtracted += ExtractorCSS.processStyleCode( + this, curi, value); + + } else if (attr.start(12) > -1) { + // METHOD + method = value; + // form processing finished at end (after ACTION also collected) + } else if (attr.start(13) > -1) { + // any other attribute + // ignore for now + // could probe for path- or script-looking strings, but + // those should be vanishingly rare in other attributes, + // and/or symptomatic of page bugs + } + } + TextUtils.recycleMatcher(attr); + + // handle codebase/resources + if (resources != null) { + Iterator iter = resources.iterator(); + UURI codebaseURI = null; + String res = null; + try { + if (codebase != null) { + // TODO: Pass in the charset. + codebaseURI = UURIFactory. + getInstance(curi.getUURI(), codebase); + } + while(iter.hasNext()) { + res = iter.next().toString(); + res = (String) TextUtils.unescapeHtml(res); + if (codebaseURI != null) { + res = codebaseURI.resolve(res).toString(); + } + processEmbed(curi, res, element); // TODO: include attribute too + } + } catch (URIException e) { + curi.getNonFatalFailures().add(e); + } catch (IllegalArgumentException e) { + DevUtils.logger.log(Level.WARNING, "processGeneralTag()\n" + + "codebase=" + codebase + " res=" + res + "\n" + + DevUtils.extraInfo(), e); + } + } + + // finish handling form action, now method is available + if(action != null) { + if(method == null || "GET".equalsIgnoreCase(method.toString()) + || ! getExtractOnlyFormGets()) { + processLink(curi, action, actionContext); + } + } + } + + + /** + * Extract the (java)script source in the given CharSequence. + * + * @param curi source CrawlURI + * @param cs CharSequence of javascript code + */ + protected void processScriptCode(ProcessorURI curi, CharSequence cs) { + if (getExtractJavascript()) { + this.numberOfLinksExtracted += + ExtractorJS.considerStrings(this, curi, cs, false); + } + } + + static final String JAVASCRIPT = "(?i)^javascript:.*"; + + /** + * Handle generic HREF cases. + * + * @param curi + * @param value + * @param context + */ + protected void processLink(ProcessorURI curi, final CharSequence value, + CharSequence context) { + if (TextUtils.matches(JAVASCRIPT, value)) { + processScriptCode(curi, value. subSequence(11, value.length())); + } else { + if (logger.isLoggable(Level.FINEST)) { + logger.finest("link: " + value.toString() + " from " + curi); + } + addLinkFromString(curi, + (value instanceof String)? + (String)value: value.toString(), + context, Hop.NAVLINK); + this.numberOfLinksExtracted++; + } + } + + private void addLinkFromString(ProcessorURI curi, String uri, + CharSequence context, Hop hop) { + try { + // We do a 'toString' on context because its a sequence from + // the underlying ReplayCharSequence and the link its about + // to become a part of is expected to outlive the current + // ReplayCharSequence. + HTMLLinkContext hc = new HTMLLinkContext(context.toString()); + int max = getExtractorParameters().getMaxOutlinks(); + Link.addRelativeToBase(curi, max, uri, hc, hop); + } catch (URIException e) { + logUriError(e, curi.getUURI(), uri); + } + } + + protected final void processEmbed(ProcessorURI curi, CharSequence value, + CharSequence context) { + processEmbed(curi, value, context, Hop.EMBED); + } + + protected void processEmbed(ProcessorURI curi, final CharSequence value, + CharSequence context, Hop hop) { + if (logger.isLoggable(Level.FINEST)) { + logger.finest("embed (" + hop.getHopChar() + "): " + value.toString() + + " from " + curi); + } + addLinkFromString(curi, + (value instanceof String)? + (String)value: value.toString(), + context, hop); + this.numberOfLinksExtracted++; + } + + + protected boolean shouldExtract(ProcessorURI uri) { + if (getIgnoreUnexpectedHtml()) { + try { + // HTML was not expected (eg a GIF was expected) so ignore + // (as if a soft 404) + if (!isHtmlExpectedHere(uri)) { + return false; + } + } catch (URIException e) { + logger.severe("Failed expectedHTML test: " + e.getMessage()); + // assume it's okay to extract + } + } + + String mime = uri.getContentType().toLowerCase(); + if (mime.startsWith("text/html")) { + return true; + } + if (mime.startsWith("application/xhtml")) { + return true; + } + + return false; + } + + + public boolean innerExtract(ProcessorURI curi) { + this.numberOfCURIsHandled++; + + ReplayCharSequence cs = null; + + try { + cs = curi.getRecorder().getReplayCharSequence(); + } catch (IOException e) { + curi.getNonFatalFailures().add(e); + //addLocalizedError(e, + // "Failed get of replay char sequence " + curi.toString() + + // " " + e.getMessage()); + logger.log(Level.SEVERE,"Failed get of replay char sequence in " + + Thread.currentThread().getName(), e); + } + + if (cs == null) { + return false; + } + + // We have a ReplayCharSequence open. Wrap all in finally so we + // for sure close it before we leave. + try { + // Extract all links from the charsequence + extract(curi, cs); + // Set flag to indicate that link extraction is completed. + return true; + } finally { + if (cs != null) { + try { + cs.close(); + } catch (IOException ioe) { + logger.warning(TextUtils.exceptionToString( + "Failed close of ReplayCharSequence.", ioe)); + } + } + } + } + + /** + * Run extractor. + * This method is package visible to ease testing. + * @param curi ProcessorURI we're processing. + * @param cs Sequence from underlying ReplayCharSequence. This + * is TRANSIENT data. Make a copy if you want the data to live outside + * of this extractors' lifetime. + */ + void extract(ProcessorURI curi, CharSequence cs) { + Matcher tags = relevantTagExtractor.matcher(cs); + while(tags.find()) { + if(Thread.interrupted()){ + break; + } + if (tags.start(8) > 0) { + // comment match + // for now do nothing + } else if (tags.start(7) > 0) { + // match + int start = tags.start(5); + int end = tags.end(5); + assert start >= 0: "Start is: " + start + ", " + curi; + assert end >= 0: "End is :" + end + ", " + curi; + if (processMeta(curi, + cs.subSequence(start, end))) { + + // meta tag included NOFOLLOW; abort processing + break; + } + } else if (tags.start(5) > 0) { + // generic match + int start5 = tags.start(5); + int end5 = tags.end(5); + assert start5 >= 0: "Start is: " + start5 + ", " + curi; + assert end5 >= 0: "End is :" + end5 + ", " + curi; + int start6 = tags.start(6); + int end6 = tags.end(6); + assert start6 >= 0: "Start is: " + start6 + ", " + curi; + assert end6 >= 0: "End is :" + end6 + ", " + curi; + processGeneralTag(curi, + cs.subSequence(start6, end6), + cs.subSequence(start5, end5)); + + } else if (tags.start(1) > 0) { + //

      14. sw1$r^iCpAn>N=<2C_PbJ<2}*iTUyiUsv-j#gzK}O^%T({ixyQ?kp$a;nOrXQl zEOKExwkp?=n$axlYUeuYkPt;BaOO6)k#5M_PX^kkB zhs1hruNAW)2WzfSOb(p%O4kzOt*DZB;-B5taKc_u%n*N0o%CY0b6bU>b?4TDOu0IX zK>G0JP*LJ_9!iMON3RYM%~OVdr-j8S&Sg2-g=aDHZ{DpT<+b zSL9m$R6cG#_dx!}$IW$noC_d!O8M;3?!7!vba8|sh&0uSC1C0-S{MJPx^_kzfieI9 z%I(?(jRAPl1UUr(qd363`v;Bq{>j^F$q(U@%qv6hfRRW96aX0Dg$OwLhWK~|djTOJ z38X+ccmX&-1R?`Thb%+TPzR_u5Eu2|{BvE^B3?X&HLIcI2C`Xv3ZH}M+iARd_F8XH zhtI#K)Xz_fqWfde?;vC}j{jxW8j3R%dWIk&ve%H$%aV(d24^;=!_ zl<8+Lf4>f=z(A-k9j5J?+KoF@=k$*2jJJcZi&ogbTg#82czsSbF4{!|Z>Ibn#Q%4# zH>kB-U1&FsEtvomYj_3MJ_{3}(L|UB{k{hsxEBL=p|jGRSe_eD@?B71M=h`eiX5E7 zSnxh2SCplifM5tEe|(FvL>yI>Ji}~&>-LMOc(Y4icKkFw?t?pI2_gPzpJa?Hy`fsK zb@UHPfXB3Ho$=eC0{8+y za1r=}KyaB$OfrZBJ5V&2kUW6M|4lTW0zV9RgwtWW97VtoFfn6A9@D<4y;8@r5^y(~ z19ixUtpt(|sG5`}IozOtMgcCfcNvWY5Ga6fVW9sJJY3=*G)H*y@cg3?_(yQ91;DjE pAT-R|+cz{+<03gIkgIFt6XA}k5H8fA_e##(yl~w4zX66DLnFkgb`1ak literal 0 HcmV?d00001 diff --git a/engine/testdata/selftest/FlashParseSelfTest/htdocs/success.html b/engine/testdata/selftest/FlashParseSelfTest/htdocs/success.html new file mode 100644 index 00000000..4ac612f8 --- /dev/null +++ b/engine/testdata/selftest/FlashParseSelfTest/htdocs/success.html @@ -0,0 +1 @@ +This page should be found by parsing a swf. diff --git a/engine/testdata/selftest/FormAuthSelfTest/htdocs/failure.html b/engine/testdata/selftest/FormAuthSelfTest/htdocs/failure.html new file mode 100644 index 00000000..891cbb5a --- /dev/null +++ b/engine/testdata/selftest/FormAuthSelfTest/htdocs/failure.html @@ -0,0 +1,9 @@ +index + +Link 1 + + Link 2 + + Link 3 + +Secure \ No newline at end of file diff --git a/engine/testdata/selftest/FormAuthSelfTest/htdocs/index.html b/engine/testdata/selftest/FormAuthSelfTest/htdocs/index.html new file mode 100644 index 00000000..891cbb5a --- /dev/null +++ b/engine/testdata/selftest/FormAuthSelfTest/htdocs/index.html @@ -0,0 +1,9 @@ +index + +Link 1 + + Link 2 + + Link 3 + +Secure \ No newline at end of file diff --git a/engine/testdata/selftest/FormAuthSelfTest/htdocs/link1.html b/engine/testdata/selftest/FormAuthSelfTest/htdocs/link1.html new file mode 100644 index 00000000..e9521a25 --- /dev/null +++ b/engine/testdata/selftest/FormAuthSelfTest/htdocs/link1.html @@ -0,0 +1,7 @@ +index + +Link 1 + + Link 2 + + Link 3 \ No newline at end of file diff --git a/engine/testdata/selftest/FormAuthSelfTest/htdocs/link2.html b/engine/testdata/selftest/FormAuthSelfTest/htdocs/link2.html new file mode 100644 index 00000000..e9521a25 --- /dev/null +++ b/engine/testdata/selftest/FormAuthSelfTest/htdocs/link2.html @@ -0,0 +1,7 @@ +index + +Link 1 + + Link 2 + + Link 3 \ No newline at end of file diff --git a/engine/testdata/selftest/FormAuthSelfTest/htdocs/link3.html b/engine/testdata/selftest/FormAuthSelfTest/htdocs/link3.html new file mode 100644 index 00000000..e9521a25 --- /dev/null +++ b/engine/testdata/selftest/FormAuthSelfTest/htdocs/link3.html @@ -0,0 +1,7 @@ +index + +Link 1 + + Link 2 + + Link 3 \ No newline at end of file diff --git a/engine/testdata/selftest/FormAuthSelfTest/htdocs/success.html b/engine/testdata/selftest/FormAuthSelfTest/htdocs/success.html new file mode 100644 index 00000000..891cbb5a --- /dev/null +++ b/engine/testdata/selftest/FormAuthSelfTest/htdocs/success.html @@ -0,0 +1,9 @@ +index + +Link 1 + + Link 2 + + Link 3 + +Secure \ No newline at end of file diff --git a/engine/testdata/selftest/FramesSelfTestCase/htdocs/index.html b/engine/testdata/selftest/FramesSelfTestCase/htdocs/index.html new file mode 100644 index 00000000..3162bae1 --- /dev/null +++ b/engine/testdata/selftest/FramesSelfTestCase/htdocs/index.html @@ -0,0 +1,27 @@ + + + Frames Self Test + + + + + + + + + + + + + <body> + <p>This content will be seen if your browser/crawler/whatever is + unable to recognize frames. + <a href="noframe.html">No frames</a>. + </p> + </body> + + diff --git a/engine/testdata/selftest/FramesSelfTestCase/htdocs/leftframe.html b/engine/testdata/selftest/FramesSelfTestCase/htdocs/leftframe.html new file mode 100644 index 00000000..b173d008 --- /dev/null +++ b/engine/testdata/selftest/FramesSelfTestCase/htdocs/leftframe.html @@ -0,0 +1,10 @@ + + + Left Frame + + +