Merge pull request #63 from adam-miller/ExtractorPDFContent

Contrib - ExtractorPDFContent
This commit is contained in:
Noah Levitt committed 2014-05-13 14:09:20 -07:00
commit 38cd14ecac
7 files changed
+311

No files matched your search

+5
View File
@@ -59,6 +59,11 @@
<artifactId>amqp-client</artifactId>
<version>3.2.1</version>
</dependency>
<dependency>
<groupId>com.itextpdf</groupId>
<artifactId>itextpdf</artifactId>
<version>5.5.0</version>
</dependency>
</dependencies>
<repositories>
<repository>
@@ -0,0 +1,185 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.extractor;
import java.io.IOException;
import java.util.ArrayList;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.httpclient.URIException;
import org.archive.modules.CrawlURI;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
import com.itextpdf.text.pdf.PdfReader;
import com.itextpdf.text.pdf.parser.PdfReaderContentParser;
import com.itextpdf.text.pdf.parser.SimpleTextExtractionStrategy;
import com.itextpdf.text.pdf.parser.TextExtractionStrategy;
/**
* PDF Content Extractor. This will parse the text content of a PDF and apply a
* regex to search for links within the body of the text.
*
* Requires itextpdf jar: http://repo1.maven.org/maven2/com/itextpdf/itextpdf/5.5.0/itextpdf-5.5.0.jar
*
* @contributor adam
*/
public class ExtractorPDFContent extends ContentExtractor {
@SuppressWarnings("unused")
private static final long serialVersionUID = 3L;
private static final Logger LOGGER =
Logger.getLogger(ExtractorPDFContent.class.getName());
public static final Pattern URLPattern = Pattern.compile(
"(?i)\\(?(https?):\\/\\/"+ // protocol
"(([a-z0-9$_\\.\\+!\\*\\'\\(\\),;\\?&=-]|%[0-9a-f]{2})+"+ // username
"(:([a-z0-9$_\\.\\+!\\*\\'\\(\\),;\\?&=-]|%[0-9a-f]{2})+)?"+ // password
"@)?(?"+ // auth requires @
")((([a-z0-9]\\.|[a-z0-9][a-z0-9-]*[a-z0-9]\\.)*"+ // domain segments AND
"[a-z][a-z0-9-]*[a-z0-9]"+ // top level domain OR
"|((\\d|[1-9]\\d|1\\d{2}|2[0-4][0-9]|25[0-5])\\.){3}"+
"(\\d|[1-9]\\d|1\\d{2}|2[0-4][0-9]|25[0-5])"+ // IP address
")(:\\d+)?)"+ // port
"(((\\/+([a-z0-9$_\\.\\+!\\*\\'\\(\\),;:@&=-]|%[0-9a-f]{2})*)*"+ // path
"(\\?([a-z0-9$_\\.\\+!\\*\\'\\(\\),;:@&=-]|%[0-9a-f]{2})*)?)?)?"+ // query string
"(\\n(?!http://)"+ // possible newline (seems to happen in pdfs)
"((\\/)?([a-z0-9$_\\.\\+!\\*\\'\\(\\),;:@&=-]|%[0-9a-f]{2})*)*"+ // continue possible path
"(\\?([a-z0-9$_\\.\\+!\\*\\'\\(\\),;:@&=-]|%[0-9a-f]{2})*)?"+ // or possible query
")?");
/**
* The maximum size of PDF files to consider. PDFs larger than this
* maximum will not be searched for links.
*/
{
setMaxSizeToParse(10*1024*1024L); // 10MB
}
public long getMaxSizeToParse() {
return (Long) kp.get("maxSizeToParse");
}
public void setMaxSizeToParse(long threshold) {
kp.put("maxSizeToParse",threshold);
}
public ExtractorPDFContent() {
}
protected boolean innerExtract(CrawlURI curi){
PdfReader documentReader;
ArrayList<String> uris = new ArrayList<String>();
try {
documentReader = new PdfReader(curi.getRecorder().getContentReplayInputStream());
for(int i=1; i<= documentReader.getNumberOfPages(); i++) { //Page numbers start at 1
String pageParseText = extractPageText(documentReader,i);
Matcher matcher = URLPattern.matcher(pageParseText);
while(matcher.find()) {
String prospectiveURL = pageParseText.substring(matcher.start(),matcher.end()).trim();
//handle URLs wrapped in parentheses
if(prospectiveURL.startsWith("(")) {
prospectiveURL=prospectiveURL.substring(1,prospectiveURL.length());
if(prospectiveURL.endsWith(")"))
prospectiveURL=prospectiveURL.substring(0,prospectiveURL.length()-1);
}
uris.add(prospectiveURL);
//parsetext URLs tend to end in a '.' if they are in a sentence, queue without trailing '.'
if(prospectiveURL.endsWith(".") && prospectiveURL.length()>2)
uris.add(prospectiveURL.substring(0, prospectiveURL.length()-1));
//Full regex allows newlines which seem to be common, also add match without newline in case we are wrong
if(matcher.group(19)!=null) {
String alternateURL = matcher.group(1)+"://"+(matcher.group(2)!=null?matcher.group(2):"")+matcher.group(6)+matcher.group(13);
//Again, handle URLs wrapped in parentheses
if(prospectiveURL.startsWith("(") && alternateURL.endsWith(")"))
alternateURL=alternateURL.substring(0,alternateURL.length()-1);
uris.add(alternateURL);
}
}
}
} catch (IOException e) {
curi.getNonFatalFailures().add(e);
return false;
} catch (RuntimeException e) {
curi.getNonFatalFailures().add(e);
return false;
}
if (uris.size()<1) {
return true;
}
for (String uri: uris) {
try {
UURI src = curi.getUURI();
UURI dest = UURIFactory.getInstance(uri);
LinkContext lc = LinkContext.NAVLINK_MISC;
Hop hop = Hop.NAVLINK;
Link out = new Link(src, dest, lc, hop);
curi.getOutLinks().add(out);
} catch (URIException e1) {
logUriError(e1, curi.getUURI(), uri);
}
}
numberOfLinksExtracted.addAndGet(uris.size());
LOGGER.fine(curi+" has "+uris.size()+" links.");
// Set flag to indicate that link extraction is completed.
return true;
}
public String extractPageText(PdfReader documentReader, int pageNum){
String content ="";
PdfReaderContentParser parser = new PdfReaderContentParser(documentReader);
TextExtractionStrategy strat;
try {
strat = parser.processContent(pageNum, new SimpleTextExtractionStrategy());
content = strat.getResultantText();
} catch (IOException e) {
LOGGER.log(Level.WARNING, "Failed to parse pdf text in "
+ Thread.currentThread().getName(), e);
}
return content;
}
@Override
protected boolean shouldExtract(CrawlURI uri) {
long max = getMaxSizeToParse();
if (uri.getRecorder().getRecordedInput().getSize() > max) {
return false;
}
String ct = uri.getContentType();
return (ct != null) && (ct.startsWith("application/pdf"));
}
}
@@ -0,0 +1,121 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.extractor;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.util.HashSet;
import java.util.Set;
import org.apache.commons.httpclient.URIException;
import org.archive.modules.CrawlURI;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
import org.archive.util.Recorder;
public class ExtractorPDFContentTest extends ContentExtractorTestBase {
protected static final String TEST_RESOURCE_FILE_1 = "ExtractorPDFContentTest1.pdf";
protected static final String TEST_RESOURCE_FILE_2 = "ExtractorPDFContentTest2.pdf";
protected static final String TEST_RESOURCE_FILE_3 = "ExtractorPDFContentTest3.pdf";
protected static final String TEST_RESOURCE_FILE_4 = "ExtractorPDFContentTest4.pdf";
public void testA() throws URIException, UnsupportedEncodingException, IOException, InterruptedException{
CrawlURI testUri = createTestUri("http://www.example.com/fake.pdf", TEST_RESOURCE_FILE_1);
extractor.process(testUri);
Set<Link> expected = makeLinkSet(testUri, new String[]{"http://www.businessdictionary.com/definition/supervisor.html","http://management.about.com/od/policiesandprocedures/g/supervisor1.html"});
assertTrue(testUri.getOutLinks().containsAll(expected));
}
public void testEndingInDot() throws URIException, UnsupportedEncodingException, IOException, InterruptedException{
CrawlURI testUri = createTestUri("http://www.example.com/fake.pdf", TEST_RESOURCE_FILE_2);
extractor.process(testUri);
Set<Link> expected = makeLinkSet(testUri, new String[]{"http://www.fec.gov/data/CommitteeSummary.do",
"http://www.opensecrets.org/bigpicture/elec_stats.php",
"http://www.opensecrets.org/pacs"});
assertTrue(testUri.getOutLinks().containsAll(expected));
}
public void testUnderscoreInURL() throws URIException, UnsupportedEncodingException, IOException, InterruptedException{
CrawlURI testUri = createTestUri("http://www.example.com/fake.pdf", TEST_RESOURCE_FILE_3);
extractor.process(testUri);
Set<Link> expected = makeLinkSet(testUri, new String[]{"http://www.dot.gov/sites/dot.dev/files/docs/2014_February_ATCR.pdf"});
assertTrue(testUri.getOutLinks().containsAll(expected));
}
public void testParenthesis() throws URIException, UnsupportedEncodingException, IOException, InterruptedException{
CrawlURI testUri = createTestUri("http://www.example.com/fake.pdf", TEST_RESOURCE_FILE_4);
extractor.process(testUri);
Set<Link> expected = makeLinkSet(testUri, new String[]{"http://www.unisys.com","http://www.myserver.mycorp.com/images/exttest.jpg","http://www.adobe.com/intro?100,200","http://www.w3.org/1999/xhtml","http://www.xfa.org/schema/xfa-data/1.0","http://www.adobe.com","http://www.adobe.com/getacro.gif","http://www.example.com/testOpeningParen"});
assertTrue(testUri.getOutLinks().containsAll(expected));
}
public void testNewlineSeparatedURIs() throws URIException, UnsupportedEncodingException, IOException, InterruptedException{
CrawlURI testUri = createTestUri("http://www.example.com/fake.pdf", TEST_RESOURCE_FILE_4);
extractor.process(testUri);
Set<Link> expected = makeLinkSet(testUri, new String[]{"http://www.unisys.com","http://www.myserver.mycorp.com/images/exttest.jpg","http://www.example.com/test","http://www.adobe.com/intro?100,200","http://www.w3.org/1999/xhtml","http://www.xfa.org/schema/xfa-data/1.0","http://www.adobe.com","http://www.adobe.com/getacro.gif"});
assertTrue(testUri.getOutLinks().containsAll(expected));
}
@Override
protected Extractor makeExtractor() {
ExtractorPDFContent result = new ExtractorPDFContent();
UriErrorLoggerModule ulm = new UnitTestUriLoggerModule();
result.setLoggerModule(ulm);
return (Extractor)result;
}
private Set<Link> makeLinkSet(CrawlURI sourceUri, String[] urlStrs) throws URIException {
HashSet<Link> linkSet = new HashSet<Link>();
for (String urlStr : urlStrs) {
linkSet.add(new Link(sourceUri.getUURI(),
UURIFactory.getInstance(urlStr),
HTMLLinkContext.NAVLINK_MISC, Hop.NAVLINK)
);
}
return linkSet;
}
private CrawlURI createTestUri(String urlStr, String resourceFileName) throws URIException,
UnsupportedEncodingException, IOException {
UURI testUuri = UURIFactory.getInstance(urlStr);
CrawlURI testUri = new CrawlURI(testUuri, null, null, LinkContext.NAVLINK_MISC);
File temp = File.createTempFile("test", ".tmp");
Recorder recorder = new Recorder(temp, 1024, 1024);
InputStream is = recorder.inputWrap(ExtractorPDFContentTest.class.getClassLoader().getResourceAsStream(resourceFileName));
recorder.markContentBegin();
for(int x = is.read(); x>=0; x=is.read());
is.close();
testUri.setContentType("application/pdf");
testUri.setFetchStatus(200);
testUri.setRecorder(recorder);
testUri.setContentSize(recorder.getResponseContentLength());
return testUri;
}
}