From 410e1709b7872b0c58b7af9e719c592eb1d336f1 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Fri, 18 Jul 2014 19:48:15 -0700 Subject: [PATCH] ExtractorHTML when a/@href links include the attribute data-remote="true", include that info the viaContext; and in FetchHTTPRequest, when a CrawlURI has data-remote="true", add the http header "X-Requested-With: XMLHttpRequest"; data-remote is a jquery thing that means the url can be requested via ajax in a browser... some urls work wrong without the X-Requested-With header... this is not a great place for a non-generic thing like this, but where else could it go... --- .../org/archive/modules/extractor/ExtractorHTML.java | 11 ++++++++--- .../org/archive/modules/fetcher/FetchHTTPRequest.java | 7 ++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index c5f710c8..0b90eee8 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -406,9 +406,15 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean CharSequence attrName = cs.subSequence(attr.start(1),attr.end(1)); value = TextUtils.unescapeHtml(value); if (attr.start(2) > -1) { + CharSequence context; // HREF - CharSequence context = elementContext(element, attr.group(2)); - if(elementStr.equalsIgnoreCase(LINK)) { + if ("a".equals(element) && TextUtils.matches("(?i).*data-remote\\s*=\\s*([\"'])true.*\\1", cs)) { + context = "a[data-remote='true']/@href"; + } else { + context = elementContext(element, attr.group(2)); + } + + if ("a[data-remote='true']/@href".equals(context) || elementStr.equalsIgnoreCase(LINK)) { // elements treated as embeds (css, ico, etc) processEmbed(curi, value, context); } else { @@ -996,6 +1002,5 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean public static CharSequence elementContext(CharSequence element, CharSequence attribute) { return attribute == null? "": element + "/@" + attribute; } - } diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java index 19fede1c..de374173 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java @@ -256,7 +256,12 @@ class FetchHTTPRequest { logger.warning("Invalid accept header: " + headerString); } } - + + if (curi.getViaContext() != null + && "a[data-remote='true']/@href".equals(curi.getViaContext().toString())) { + request.addHeader("X-Requested-With", "XMLHttpRequest"); + } + @SuppressWarnings("unchecked") Map uriCustomHeaders = (Map) curi.getData().get("customHttpRequestHeaders"); if (uriCustomHeaders != null) {