lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-20 12:46:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #87 from vonrosen/ari-3977

Browse Source 
Allow xml extractor to handle urls in CDATA.
This commit is contained in:
Noah Levitt
2014-09-10 12:42:34 -07:00
parent 066cd17b5e 27d632159d
commit e2064237cf
2 changed files with 57 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java
+2 -1
View File
@@ -56,7 +56,8 @@ public class ExtractorXML extends ContentExtractor {
Logger.getLogger(ExtractorXML.class.getName());
static final Pattern XML_URI_EXTRACTOR = Pattern
.compile("(?s)[\"\'>]\\s*([^<>\\s'\"@]+)\\s*[\"\'<]");
.compile("(?s)[\"\'>]\\s*(?:<!\\[CDATA\\[)?([^<>\\s\'\"@]+)\\s*(?:\\]\\]>)?[\"\'<]");
// GROUPS:
// (G1) possible uri
modules/src/test/java/org/archive/modules/extractor/ExtractorXMLTest.java
+55 -3
View File
@@ -18,15 +18,67 @@
*/
package org.archive.modules.extractor;
import org.archive.modules.ProcessorTestBase;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.archive.modules.CrawlMetadata;
import org.archive.modules.CrawlURI;
import org.archive.modules.extractor.StringExtractorTestBase.TestData;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
import org.archive.util.Recorder;
/**
* Unit test for {@link ExtractorXML}.
*
* @author pjack
*/
public class ExtractorXMLTest extends ProcessorTestBase {
public class ExtractorXMLTest extends StringExtractorTestBase {
// TODO TESTME!
final public static String[] VALID_TEST_DATA = new String[] {
"<link>http://conservation.org</link>",
"http://conservation.org",
"<CI:imagePath><![CDATA[http://sp10.conservation.org/CIFMGPhotos/790x444_skerry_gallery_02.jpg]]></CI:imagePath>",
"http://sp10.conservation.org/CIFMGPhotos/790x444_skerry_gallery_02.jpg",
};
@Override
protected String[] getValidTestData() {
return VALID_TEST_DATA;
}
@Override
protected Extractor makeExtractor() {
ExtractorXML result = new ExtractorXML();
UriErrorLoggerModule ulm = new UnitTestUriLoggerModule();
result.setLoggerModule(ulm);
return result;
}
protected ExtractorXML getExtractor() {
return (ExtractorXML) extractor;
}
@Override
protected Collection<TestData> makeData(String content, String destURI)
throws Exception {
List<TestData> result = new ArrayList<TestData>();
UURI src = UURIFactory.getInstance("http://www.archive.org/start/");
CrawlURI euri = new CrawlURI(src, null, null,
LinkContext.SPECULATIVE_MISC);
Recorder recorder = createRecorder(content, "UTF-8");
euri.setContentType("text/xml");
euri.setRecorder(recorder);
euri.setContentSize(content.length());
UURI dest = UURIFactory.getInstance(destURI);
CrawlURI link = euri.createCrawlURI(dest, LinkContext.SPECULATIVE_MISC, Hop.SPECULATIVE);
result.add(new TestData(euri, link));
return result;
}
}