diff --git a/contrib/pom.xml b/contrib/pom.xml index 6e048084..a9d16633 100644 --- a/contrib/pom.xml +++ b/contrib/pom.xml @@ -59,6 +59,11 @@ amqp-client 3.2.1 + + com.itextpdf + itextpdf + 5.5.0 + diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorPDFContent.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorPDFContent.java new file mode 100644 index 00000000..082517ca --- /dev/null +++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorPDFContent.java @@ -0,0 +1,185 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.extractor; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.httpclient.URIException; +import org.archive.modules.CrawlURI; +import org.archive.net.UURI; +import org.archive.net.UURIFactory; + +import com.itextpdf.text.pdf.PdfReader; +import com.itextpdf.text.pdf.parser.PdfReaderContentParser; +import com.itextpdf.text.pdf.parser.SimpleTextExtractionStrategy; +import com.itextpdf.text.pdf.parser.TextExtractionStrategy; + +/** + * PDF Content Extractor. This will parse the text content of a PDF and apply a + * regex to search for links within the body of the text. + * + * Requires itextpdf jar: http://repo1.maven.org/maven2/com/itextpdf/itextpdf/5.5.0/itextpdf-5.5.0.jar + * + * @contributor adam + */ +public class ExtractorPDFContent extends ContentExtractor { + + @SuppressWarnings("unused") + private static final long serialVersionUID = 3L; + + private static final Logger LOGGER = + Logger.getLogger(ExtractorPDFContent.class.getName()); + + public static final Pattern URLPattern = Pattern.compile( + "(?i)\\(?(https?):\\/\\/"+ // protocol + "(([a-z0-9$_\\.\\+!\\*\\'\\(\\),;\\?&=-]|%[0-9a-f]{2})+"+ // username + "(:([a-z0-9$_\\.\\+!\\*\\'\\(\\),;\\?&=-]|%[0-9a-f]{2})+)?"+ // password + "@)?(?"+ // auth requires @ + ")((([a-z0-9]\\.|[a-z0-9][a-z0-9-]*[a-z0-9]\\.)*"+ // domain segments AND + "[a-z][a-z0-9-]*[a-z0-9]"+ // top level domain OR + "|((\\d|[1-9]\\d|1\\d{2}|2[0-4][0-9]|25[0-5])\\.){3}"+ + "(\\d|[1-9]\\d|1\\d{2}|2[0-4][0-9]|25[0-5])"+ // IP address + ")(:\\d+)?)"+ // port + "(((\\/+([a-z0-9$_\\.\\+!\\*\\'\\(\\),;:@&=-]|%[0-9a-f]{2})*)*"+ // path + "(\\?([a-z0-9$_\\.\\+!\\*\\'\\(\\),;:@&=-]|%[0-9a-f]{2})*)?)?)?"+ // query string + "(\\n(?!http://)"+ // possible newline (seems to happen in pdfs) + "((\\/)?([a-z0-9$_\\.\\+!\\*\\'\\(\\),;:@&=-]|%[0-9a-f]{2})*)*"+ // continue possible path + "(\\?([a-z0-9$_\\.\\+!\\*\\'\\(\\),;:@&=-]|%[0-9a-f]{2})*)?"+ // or possible query + ")?"); + + /** + * The maximum size of PDF files to consider. PDFs larger than this + * maximum will not be searched for links. + */ + { + setMaxSizeToParse(10*1024*1024L); // 10MB + } + public long getMaxSizeToParse() { + return (Long) kp.get("maxSizeToParse"); + } + public void setMaxSizeToParse(long threshold) { + kp.put("maxSizeToParse",threshold); + } + + + public ExtractorPDFContent() { + } + + protected boolean innerExtract(CrawlURI curi){ + PdfReader documentReader; + ArrayList uris = new ArrayList(); + + try { + documentReader = new PdfReader(curi.getRecorder().getContentReplayInputStream()); + + for(int i=1; i<= documentReader.getNumberOfPages(); i++) { //Page numbers start at 1 + String pageParseText = extractPageText(documentReader,i); + Matcher matcher = URLPattern.matcher(pageParseText); + + while(matcher.find()) { + String prospectiveURL = pageParseText.substring(matcher.start(),matcher.end()).trim(); + + //handle URLs wrapped in parentheses + if(prospectiveURL.startsWith("(")) { + prospectiveURL=prospectiveURL.substring(1,prospectiveURL.length()); + if(prospectiveURL.endsWith(")")) + prospectiveURL=prospectiveURL.substring(0,prospectiveURL.length()-1); + } + + uris.add(prospectiveURL); + + //parsetext URLs tend to end in a '.' if they are in a sentence, queue without trailing '.' + if(prospectiveURL.endsWith(".") && prospectiveURL.length()>2) + uris.add(prospectiveURL.substring(0, prospectiveURL.length()-1)); + + //Full regex allows newlines which seem to be common, also add match without newline in case we are wrong + if(matcher.group(19)!=null) { + String alternateURL = matcher.group(1)+"://"+(matcher.group(2)!=null?matcher.group(2):"")+matcher.group(6)+matcher.group(13); + + //Again, handle URLs wrapped in parentheses + if(prospectiveURL.startsWith("(") && alternateURL.endsWith(")")) + alternateURL=alternateURL.substring(0,alternateURL.length()-1); + + uris.add(alternateURL); + } + } + } + + } catch (IOException e) { + curi.getNonFatalFailures().add(e); + return false; + } catch (RuntimeException e) { + curi.getNonFatalFailures().add(e); + return false; + } + + if (uris.size()<1) { + return true; + } + + for (String uri: uris) { + try { + UURI src = curi.getUURI(); + UURI dest = UURIFactory.getInstance(uri); + LinkContext lc = LinkContext.NAVLINK_MISC; + Hop hop = Hop.NAVLINK; + Link out = new Link(src, dest, lc, hop); + curi.getOutLinks().add(out); + } catch (URIException e1) { + logUriError(e1, curi.getUURI(), uri); + } + } + + numberOfLinksExtracted.addAndGet(uris.size()); + + LOGGER.fine(curi+" has "+uris.size()+" links."); + // Set flag to indicate that link extraction is completed. + return true; + } + + public String extractPageText(PdfReader documentReader, int pageNum){ + String content =""; + PdfReaderContentParser parser = new PdfReaderContentParser(documentReader); + TextExtractionStrategy strat; + try { + strat = parser.processContent(pageNum, new SimpleTextExtractionStrategy()); + content = strat.getResultantText(); + + } catch (IOException e) { + LOGGER.log(Level.WARNING, "Failed to parse pdf text in " + + Thread.currentThread().getName(), e); + } + return content; + } + @Override + protected boolean shouldExtract(CrawlURI uri) { + long max = getMaxSizeToParse(); + if (uri.getRecorder().getRecordedInput().getSize() > max) { + return false; + } + + String ct = uri.getContentType(); + return (ct != null) && (ct.startsWith("application/pdf")); + } +} diff --git a/contrib/src/test/java/org/archive/modules/extractor/ExtractorPDFContentTest.java b/contrib/src/test/java/org/archive/modules/extractor/ExtractorPDFContentTest.java new file mode 100644 index 00000000..32701bfa --- /dev/null +++ b/contrib/src/test/java/org/archive/modules/extractor/ExtractorPDFContentTest.java @@ -0,0 +1,121 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.extractor; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.util.HashSet; +import java.util.Set; + +import org.apache.commons.httpclient.URIException; +import org.archive.modules.CrawlURI; +import org.archive.net.UURI; +import org.archive.net.UURIFactory; +import org.archive.util.Recorder; + +public class ExtractorPDFContentTest extends ContentExtractorTestBase { + + protected static final String TEST_RESOURCE_FILE_1 = "ExtractorPDFContentTest1.pdf"; + protected static final String TEST_RESOURCE_FILE_2 = "ExtractorPDFContentTest2.pdf"; + protected static final String TEST_RESOURCE_FILE_3 = "ExtractorPDFContentTest3.pdf"; + protected static final String TEST_RESOURCE_FILE_4 = "ExtractorPDFContentTest4.pdf"; + + + public void testA() throws URIException, UnsupportedEncodingException, IOException, InterruptedException{ + CrawlURI testUri = createTestUri("http://www.example.com/fake.pdf", TEST_RESOURCE_FILE_1); + extractor.process(testUri); + + Set expected = makeLinkSet(testUri, new String[]{"http://www.businessdictionary.com/definition/supervisor.html","http://management.about.com/od/policiesandprocedures/g/supervisor1.html"}); + assertTrue(testUri.getOutLinks().containsAll(expected)); + } + public void testEndingInDot() throws URIException, UnsupportedEncodingException, IOException, InterruptedException{ + CrawlURI testUri = createTestUri("http://www.example.com/fake.pdf", TEST_RESOURCE_FILE_2); + extractor.process(testUri); + + Set expected = makeLinkSet(testUri, new String[]{"http://www.fec.gov/data/CommitteeSummary.do", + "http://www.opensecrets.org/bigpicture/elec_stats.php", + "http://www.opensecrets.org/pacs"}); + assertTrue(testUri.getOutLinks().containsAll(expected)); + } + public void testUnderscoreInURL() throws URIException, UnsupportedEncodingException, IOException, InterruptedException{ + CrawlURI testUri = createTestUri("http://www.example.com/fake.pdf", TEST_RESOURCE_FILE_3); + extractor.process(testUri); + + Set expected = makeLinkSet(testUri, new String[]{"http://www.dot.gov/sites/dot.dev/files/docs/2014_February_ATCR.pdf"}); + assertTrue(testUri.getOutLinks().containsAll(expected)); + } + public void testParenthesis() throws URIException, UnsupportedEncodingException, IOException, InterruptedException{ + CrawlURI testUri = createTestUri("http://www.example.com/fake.pdf", TEST_RESOURCE_FILE_4); + extractor.process(testUri); + + Set expected = makeLinkSet(testUri, new String[]{"http://www.unisys.com","http://www.myserver.mycorp.com/images/exttest.jpg","http://www.adobe.com/intro?100,200","http://www.w3.org/1999/xhtml","http://www.xfa.org/schema/xfa-data/1.0","http://www.adobe.com","http://www.adobe.com/getacro.gif","http://www.example.com/testOpeningParen"}); + assertTrue(testUri.getOutLinks().containsAll(expected)); + } + public void testNewlineSeparatedURIs() throws URIException, UnsupportedEncodingException, IOException, InterruptedException{ + CrawlURI testUri = createTestUri("http://www.example.com/fake.pdf", TEST_RESOURCE_FILE_4); + extractor.process(testUri); + + Set expected = makeLinkSet(testUri, new String[]{"http://www.unisys.com","http://www.myserver.mycorp.com/images/exttest.jpg","http://www.example.com/test","http://www.adobe.com/intro?100,200","http://www.w3.org/1999/xhtml","http://www.xfa.org/schema/xfa-data/1.0","http://www.adobe.com","http://www.adobe.com/getacro.gif"}); + assertTrue(testUri.getOutLinks().containsAll(expected)); + } + + + + @Override + protected Extractor makeExtractor() { + ExtractorPDFContent result = new ExtractorPDFContent(); + UriErrorLoggerModule ulm = new UnitTestUriLoggerModule(); + result.setLoggerModule(ulm); + return (Extractor)result; + } + private Set makeLinkSet(CrawlURI sourceUri, String[] urlStrs) throws URIException { + HashSet linkSet = new HashSet(); + for (String urlStr : urlStrs) { + linkSet.add(new Link(sourceUri.getUURI(), + UURIFactory.getInstance(urlStr), + HTMLLinkContext.NAVLINK_MISC, Hop.NAVLINK) + ); + } + return linkSet; + } + private CrawlURI createTestUri(String urlStr, String resourceFileName) throws URIException, + UnsupportedEncodingException, IOException { + UURI testUuri = UURIFactory.getInstance(urlStr); + CrawlURI testUri = new CrawlURI(testUuri, null, null, LinkContext.NAVLINK_MISC); + + + File temp = File.createTempFile("test", ".tmp"); + Recorder recorder = new Recorder(temp, 1024, 1024); + InputStream is = recorder.inputWrap(ExtractorPDFContentTest.class.getClassLoader().getResourceAsStream(resourceFileName)); + recorder.markContentBegin(); + for(int x = is.read(); x>=0; x=is.read()); + is.close(); + + + testUri.setContentType("application/pdf"); + testUri.setFetchStatus(200); + testUri.setRecorder(recorder); + testUri.setContentSize(recorder.getResponseContentLength()); + return testUri; + } + + +} diff --git a/contrib/src/test/resources/ExtractorPDFContentTest1.pdf b/contrib/src/test/resources/ExtractorPDFContentTest1.pdf new file mode 100644 index 00000000..c7e62df7 Binary files /dev/null and b/contrib/src/test/resources/ExtractorPDFContentTest1.pdf differ diff --git a/contrib/src/test/resources/ExtractorPDFContentTest2.pdf b/contrib/src/test/resources/ExtractorPDFContentTest2.pdf new file mode 100644 index 00000000..42e2af76 Binary files /dev/null and b/contrib/src/test/resources/ExtractorPDFContentTest2.pdf differ diff --git a/contrib/src/test/resources/ExtractorPDFContentTest3.pdf b/contrib/src/test/resources/ExtractorPDFContentTest3.pdf new file mode 100644 index 00000000..eda6dae4 Binary files /dev/null and b/contrib/src/test/resources/ExtractorPDFContentTest3.pdf differ diff --git a/contrib/src/test/resources/ExtractorPDFContentTest4.pdf b/contrib/src/test/resources/ExtractorPDFContentTest4.pdf new file mode 100644 index 00000000..41e2bbaa Binary files /dev/null and b/contrib/src/test/resources/ExtractorPDFContentTest4.pdf differ