lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-23 22:26:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix for [HER-1824] ExtractorXML doesn't properly unescape

Browse Source 
* ExtractorXML.java
    processXML() - unescape extracted urls with commons-lang unescapeXml()
This commit is contained in:
nlevitt
2010-09-23 02:03:48 +00:00
parent 7909954b8f
commit 63c8de4045
1 changed files with 2 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java
+2 -6
View File
@@ -24,10 +24,10 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.lang.StringEscapeUtils;
import org.archive.io.ReplayCharSequence;
import org.archive.modules.CrawlURI;
import org.archive.util.ArchiveUtils;
import org.archive.util.TextUtils;
/**
* A simple extractor which finds HTTP URIs inside XML/RSS files,
@@ -45,8 +45,6 @@ public class ExtractorXML extends ContentExtractor {
private static Logger logger =
Logger.getLogger(ExtractorXML.class.getName());
private static String ESCAPED_AMP = "&amp";
static final Pattern XML_URI_EXTRACTOR = Pattern.compile(
"(?i)[\"\'>]\\s*(http:[^\\s\"\'<>]+)\\s*[\"\'<]");
// GROUPS:
@@ -101,9 +99,7 @@ public class ExtractorXML extends ContentExtractor {
String xmlUri;
uris = XML_URI_EXTRACTOR.matcher(cs);
while (uris.find()) {
xmlUri = uris.group(1);
// TODO: Escape more HTML Entities.
xmlUri = TextUtils.replaceAll(ESCAPED_AMP, xmlUri, "&");
xmlUri = StringEscapeUtils.unescapeXml(uris.group(1));
foundLinks++;
try {
// treat as speculative, as whether context really