lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-18 19:55:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #80 from nlevitt/ari-3617-data-remote

Browse Source 
ExtractorHTML when a/@href links include the attribute data-remote="true...
This commit is contained in:
vonrosen
2014-09-04 16:08:33 -07:00
parent 25c3b3e680 410e1709b7
commit 035a7b0a08
2 changed files with 14 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java
+8 -3
View File
@@ -406,9 +406,15 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
CharSequence attrName = cs.subSequence(attr.start(1),attr.end(1));
value = TextUtils.unescapeHtml(value);
if (attr.start(2) > -1) {
CharSequence context;
// HREF
CharSequence context = elementContext(element, attr.group(2));
if(elementStr.equalsIgnoreCase(LINK)) {
if ("a".equals(element) && TextUtils.matches("(?i).*data-remote\\s*=\\s*([\"'])true.*\\1", cs)) {
context = "a[data-remote='true']/@href";
} else {
context = elementContext(element, attr.group(2));
}
if ("a[data-remote='true']/@href".equals(context) || elementStr.equalsIgnoreCase(LINK)) {
// <LINK> elements treated as embeds (css, ico, etc)
processEmbed(curi, value, context);
} else {
@@ -1004,6 +1010,5 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
public static CharSequence elementContext(CharSequence element, CharSequence attribute) {
return attribute == null? "": element + "/@" + attribute;
}
}
modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java
+6 -1
View File
@@ -256,7 +256,12 @@ class FetchHTTPRequest {
logger.warning("Invalid accept header: " + headerString);
}
}
if (curi.getViaContext() != null
&& "a[data-remote='true']/@href".equals(curi.getViaContext().toString())) {
request.addHeader("X-Requested-With", "XMLHttpRequest");
}
@SuppressWarnings("unchecked")
Map<String, String> uriCustomHeaders = (Map<String, String>) curi.getData().get("customHttpRequestHeaders");
if (uriCustomHeaders != null) {