diff --git a/contrib/pom.xml b/contrib/pom.xml
index 6e048084..a9d16633 100644
--- a/contrib/pom.xml
+++ b/contrib/pom.xml
@@ -59,6 +59,11 @@
amqp-client
3.2.1
+
+ com.itextpdf
+ itextpdf
+ 5.5.0
+
diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorPDFContent.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorPDFContent.java
new file mode 100644
index 00000000..082517ca
--- /dev/null
+++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorPDFContent.java
@@ -0,0 +1,185 @@
+/*
+ * This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ * Licensed to the Internet Archive (IA) by one or more individual
+ * contributors.
+ *
+ * The IA licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.archive.modules.extractor;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.httpclient.URIException;
+import org.archive.modules.CrawlURI;
+import org.archive.net.UURI;
+import org.archive.net.UURIFactory;
+
+import com.itextpdf.text.pdf.PdfReader;
+import com.itextpdf.text.pdf.parser.PdfReaderContentParser;
+import com.itextpdf.text.pdf.parser.SimpleTextExtractionStrategy;
+import com.itextpdf.text.pdf.parser.TextExtractionStrategy;
+
+/**
+ * PDF Content Extractor. This will parse the text content of a PDF and apply a
+ * regex to search for links within the body of the text.
+ *
+ * Requires itextpdf jar: http://repo1.maven.org/maven2/com/itextpdf/itextpdf/5.5.0/itextpdf-5.5.0.jar
+ *
+ * @contributor adam
+ */
+public class ExtractorPDFContent extends ContentExtractor {
+
+ @SuppressWarnings("unused")
+ private static final long serialVersionUID = 3L;
+
+ private static final Logger LOGGER =
+ Logger.getLogger(ExtractorPDFContent.class.getName());
+
+ public static final Pattern URLPattern = Pattern.compile(
+ "(?i)\\(?(https?):\\/\\/"+ // protocol
+ "(([a-z0-9$_\\.\\+!\\*\\'\\(\\),;\\?&=-]|%[0-9a-f]{2})+"+ // username
+ "(:([a-z0-9$_\\.\\+!\\*\\'\\(\\),;\\?&=-]|%[0-9a-f]{2})+)?"+ // password
+ "@)?(?"+ // auth requires @
+ ")((([a-z0-9]\\.|[a-z0-9][a-z0-9-]*[a-z0-9]\\.)*"+ // domain segments AND
+ "[a-z][a-z0-9-]*[a-z0-9]"+ // top level domain OR
+ "|((\\d|[1-9]\\d|1\\d{2}|2[0-4][0-9]|25[0-5])\\.){3}"+
+ "(\\d|[1-9]\\d|1\\d{2}|2[0-4][0-9]|25[0-5])"+ // IP address
+ ")(:\\d+)?)"+ // port
+ "(((\\/+([a-z0-9$_\\.\\+!\\*\\'\\(\\),;:@&=-]|%[0-9a-f]{2})*)*"+ // path
+ "(\\?([a-z0-9$_\\.\\+!\\*\\'\\(\\),;:@&=-]|%[0-9a-f]{2})*)?)?)?"+ // query string
+ "(\\n(?!http://)"+ // possible newline (seems to happen in pdfs)
+ "((\\/)?([a-z0-9$_\\.\\+!\\*\\'\\(\\),;:@&=-]|%[0-9a-f]{2})*)*"+ // continue possible path
+ "(\\?([a-z0-9$_\\.\\+!\\*\\'\\(\\),;:@&=-]|%[0-9a-f]{2})*)?"+ // or possible query
+ ")?");
+
+ /**
+ * The maximum size of PDF files to consider. PDFs larger than this
+ * maximum will not be searched for links.
+ */
+ {
+ setMaxSizeToParse(10*1024*1024L); // 10MB
+ }
+ public long getMaxSizeToParse() {
+ return (Long) kp.get("maxSizeToParse");
+ }
+ public void setMaxSizeToParse(long threshold) {
+ kp.put("maxSizeToParse",threshold);
+ }
+
+
+ public ExtractorPDFContent() {
+ }
+
+ protected boolean innerExtract(CrawlURI curi){
+ PdfReader documentReader;
+ ArrayList uris = new ArrayList();
+
+ try {
+ documentReader = new PdfReader(curi.getRecorder().getContentReplayInputStream());
+
+ for(int i=1; i<= documentReader.getNumberOfPages(); i++) { //Page numbers start at 1
+ String pageParseText = extractPageText(documentReader,i);
+ Matcher matcher = URLPattern.matcher(pageParseText);
+
+ while(matcher.find()) {
+ String prospectiveURL = pageParseText.substring(matcher.start(),matcher.end()).trim();
+
+ //handle URLs wrapped in parentheses
+ if(prospectiveURL.startsWith("(")) {
+ prospectiveURL=prospectiveURL.substring(1,prospectiveURL.length());
+ if(prospectiveURL.endsWith(")"))
+ prospectiveURL=prospectiveURL.substring(0,prospectiveURL.length()-1);
+ }
+
+ uris.add(prospectiveURL);
+
+ //parsetext URLs tend to end in a '.' if they are in a sentence, queue without trailing '.'
+ if(prospectiveURL.endsWith(".") && prospectiveURL.length()>2)
+ uris.add(prospectiveURL.substring(0, prospectiveURL.length()-1));
+
+ //Full regex allows newlines which seem to be common, also add match without newline in case we are wrong
+ if(matcher.group(19)!=null) {
+ String alternateURL = matcher.group(1)+"://"+(matcher.group(2)!=null?matcher.group(2):"")+matcher.group(6)+matcher.group(13);
+
+ //Again, handle URLs wrapped in parentheses
+ if(prospectiveURL.startsWith("(") && alternateURL.endsWith(")"))
+ alternateURL=alternateURL.substring(0,alternateURL.length()-1);
+
+ uris.add(alternateURL);
+ }
+ }
+ }
+
+ } catch (IOException e) {
+ curi.getNonFatalFailures().add(e);
+ return false;
+ } catch (RuntimeException e) {
+ curi.getNonFatalFailures().add(e);
+ return false;
+ }
+
+ if (uris.size()<1) {
+ return true;
+ }
+
+ for (String uri: uris) {
+ try {
+ UURI src = curi.getUURI();
+ UURI dest = UURIFactory.getInstance(uri);
+ LinkContext lc = LinkContext.NAVLINK_MISC;
+ Hop hop = Hop.NAVLINK;
+ Link out = new Link(src, dest, lc, hop);
+ curi.getOutLinks().add(out);
+ } catch (URIException e1) {
+ logUriError(e1, curi.getUURI(), uri);
+ }
+ }
+
+ numberOfLinksExtracted.addAndGet(uris.size());
+
+ LOGGER.fine(curi+" has "+uris.size()+" links.");
+ // Set flag to indicate that link extraction is completed.
+ return true;
+ }
+
+ public String extractPageText(PdfReader documentReader, int pageNum){
+ String content ="";
+ PdfReaderContentParser parser = new PdfReaderContentParser(documentReader);
+ TextExtractionStrategy strat;
+ try {
+ strat = parser.processContent(pageNum, new SimpleTextExtractionStrategy());
+ content = strat.getResultantText();
+
+ } catch (IOException e) {
+ LOGGER.log(Level.WARNING, "Failed to parse pdf text in "
+ + Thread.currentThread().getName(), e);
+ }
+ return content;
+ }
+ @Override
+ protected boolean shouldExtract(CrawlURI uri) {
+ long max = getMaxSizeToParse();
+ if (uri.getRecorder().getRecordedInput().getSize() > max) {
+ return false;
+ }
+
+ String ct = uri.getContentType();
+ return (ct != null) && (ct.startsWith("application/pdf"));
+ }
+}
diff --git a/contrib/src/test/java/org/archive/modules/extractor/ExtractorPDFContentTest.java b/contrib/src/test/java/org/archive/modules/extractor/ExtractorPDFContentTest.java
new file mode 100644
index 00000000..32701bfa
--- /dev/null
+++ b/contrib/src/test/java/org/archive/modules/extractor/ExtractorPDFContentTest.java
@@ -0,0 +1,121 @@
+/*
+ * This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ * Licensed to the Internet Archive (IA) by one or more individual
+ * contributors.
+ *
+ * The IA licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.archive.modules.extractor;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.httpclient.URIException;
+import org.archive.modules.CrawlURI;
+import org.archive.net.UURI;
+import org.archive.net.UURIFactory;
+import org.archive.util.Recorder;
+
+public class ExtractorPDFContentTest extends ContentExtractorTestBase {
+
+ protected static final String TEST_RESOURCE_FILE_1 = "ExtractorPDFContentTest1.pdf";
+ protected static final String TEST_RESOURCE_FILE_2 = "ExtractorPDFContentTest2.pdf";
+ protected static final String TEST_RESOURCE_FILE_3 = "ExtractorPDFContentTest3.pdf";
+ protected static final String TEST_RESOURCE_FILE_4 = "ExtractorPDFContentTest4.pdf";
+
+
+ public void testA() throws URIException, UnsupportedEncodingException, IOException, InterruptedException{
+ CrawlURI testUri = createTestUri("http://www.example.com/fake.pdf", TEST_RESOURCE_FILE_1);
+ extractor.process(testUri);
+
+ Set expected = makeLinkSet(testUri, new String[]{"http://www.businessdictionary.com/definition/supervisor.html","http://management.about.com/od/policiesandprocedures/g/supervisor1.html"});
+ assertTrue(testUri.getOutLinks().containsAll(expected));
+ }
+ public void testEndingInDot() throws URIException, UnsupportedEncodingException, IOException, InterruptedException{
+ CrawlURI testUri = createTestUri("http://www.example.com/fake.pdf", TEST_RESOURCE_FILE_2);
+ extractor.process(testUri);
+
+ Set expected = makeLinkSet(testUri, new String[]{"http://www.fec.gov/data/CommitteeSummary.do",
+ "http://www.opensecrets.org/bigpicture/elec_stats.php",
+ "http://www.opensecrets.org/pacs"});
+ assertTrue(testUri.getOutLinks().containsAll(expected));
+ }
+ public void testUnderscoreInURL() throws URIException, UnsupportedEncodingException, IOException, InterruptedException{
+ CrawlURI testUri = createTestUri("http://www.example.com/fake.pdf", TEST_RESOURCE_FILE_3);
+ extractor.process(testUri);
+
+ Set expected = makeLinkSet(testUri, new String[]{"http://www.dot.gov/sites/dot.dev/files/docs/2014_February_ATCR.pdf"});
+ assertTrue(testUri.getOutLinks().containsAll(expected));
+ }
+ public void testParenthesis() throws URIException, UnsupportedEncodingException, IOException, InterruptedException{
+ CrawlURI testUri = createTestUri("http://www.example.com/fake.pdf", TEST_RESOURCE_FILE_4);
+ extractor.process(testUri);
+
+ Set expected = makeLinkSet(testUri, new String[]{"http://www.unisys.com","http://www.myserver.mycorp.com/images/exttest.jpg","http://www.adobe.com/intro?100,200","http://www.w3.org/1999/xhtml","http://www.xfa.org/schema/xfa-data/1.0","http://www.adobe.com","http://www.adobe.com/getacro.gif","http://www.example.com/testOpeningParen"});
+ assertTrue(testUri.getOutLinks().containsAll(expected));
+ }
+ public void testNewlineSeparatedURIs() throws URIException, UnsupportedEncodingException, IOException, InterruptedException{
+ CrawlURI testUri = createTestUri("http://www.example.com/fake.pdf", TEST_RESOURCE_FILE_4);
+ extractor.process(testUri);
+
+ Set expected = makeLinkSet(testUri, new String[]{"http://www.unisys.com","http://www.myserver.mycorp.com/images/exttest.jpg","http://www.example.com/test","http://www.adobe.com/intro?100,200","http://www.w3.org/1999/xhtml","http://www.xfa.org/schema/xfa-data/1.0","http://www.adobe.com","http://www.adobe.com/getacro.gif"});
+ assertTrue(testUri.getOutLinks().containsAll(expected));
+ }
+
+
+
+ @Override
+ protected Extractor makeExtractor() {
+ ExtractorPDFContent result = new ExtractorPDFContent();
+ UriErrorLoggerModule ulm = new UnitTestUriLoggerModule();
+ result.setLoggerModule(ulm);
+ return (Extractor)result;
+ }
+ private Set makeLinkSet(CrawlURI sourceUri, String[] urlStrs) throws URIException {
+ HashSet linkSet = new HashSet();
+ for (String urlStr : urlStrs) {
+ linkSet.add(new Link(sourceUri.getUURI(),
+ UURIFactory.getInstance(urlStr),
+ HTMLLinkContext.NAVLINK_MISC, Hop.NAVLINK)
+ );
+ }
+ return linkSet;
+ }
+ private CrawlURI createTestUri(String urlStr, String resourceFileName) throws URIException,
+ UnsupportedEncodingException, IOException {
+ UURI testUuri = UURIFactory.getInstance(urlStr);
+ CrawlURI testUri = new CrawlURI(testUuri, null, null, LinkContext.NAVLINK_MISC);
+
+
+ File temp = File.createTempFile("test", ".tmp");
+ Recorder recorder = new Recorder(temp, 1024, 1024);
+ InputStream is = recorder.inputWrap(ExtractorPDFContentTest.class.getClassLoader().getResourceAsStream(resourceFileName));
+ recorder.markContentBegin();
+ for(int x = is.read(); x>=0; x=is.read());
+ is.close();
+
+
+ testUri.setContentType("application/pdf");
+ testUri.setFetchStatus(200);
+ testUri.setRecorder(recorder);
+ testUri.setContentSize(recorder.getResponseContentLength());
+ return testUri;
+ }
+
+
+}
diff --git a/contrib/src/test/resources/ExtractorPDFContentTest1.pdf b/contrib/src/test/resources/ExtractorPDFContentTest1.pdf
new file mode 100644
index 00000000..c7e62df7
Binary files /dev/null and b/contrib/src/test/resources/ExtractorPDFContentTest1.pdf differ
diff --git a/contrib/src/test/resources/ExtractorPDFContentTest2.pdf b/contrib/src/test/resources/ExtractorPDFContentTest2.pdf
new file mode 100644
index 00000000..42e2af76
Binary files /dev/null and b/contrib/src/test/resources/ExtractorPDFContentTest2.pdf differ
diff --git a/contrib/src/test/resources/ExtractorPDFContentTest3.pdf b/contrib/src/test/resources/ExtractorPDFContentTest3.pdf
new file mode 100644
index 00000000..eda6dae4
Binary files /dev/null and b/contrib/src/test/resources/ExtractorPDFContentTest3.pdf differ
diff --git a/contrib/src/test/resources/ExtractorPDFContentTest4.pdf b/contrib/src/test/resources/ExtractorPDFContentTest4.pdf
new file mode 100644
index 00000000..41e2bbaa
Binary files /dev/null and b/contrib/src/test/resources/ExtractorPDFContentTest4.pdf differ