diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index 3562611d..d8f709b4 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -436,7 +436,16 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean CharSequence value = cs.subSequence(start, end); CharSequence attrName = cs.subSequence(attr.start(1),attr.end(1)); value = TextUtils.unescapeHtml(value); - if (attr.start(2) > -1) { + if (value.length() == getMaxAttributeValLength() && end < cs.length()) { + char nextChar = cs.charAt(end); + // Check if it's really the end of the string + if (nextChar != '"' && nextChar != '\'' && !Character.isWhitespace(nextChar)) { + // Do nothing, truncated value + if (logger.isLoggable(Level.FINE)) { + logger.fine("Truncated value: " + value); + } + } + } else if (attr.start(2) > -1) { CharSequence context; // HREF if ("a".equals(element) && TextUtils.matches("(?i).*data-remote\\s*=\\s*([\"'])true.*\\1", cs)) { diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java index 1d5405bf..4be57e8c 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java @@ -275,6 +275,14 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { getExtractor().extract(curi, cs); assertEquals(0, curi.getOutLinks().size()); } + + @Test + public void testTooLongUrisAreIgnored() throws URIException { + CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://www.example.com")); + CharSequence cs = ""; + getExtractor().extract(curi, cs); + assertEquals(0, curi.getOutLinks().size()); + } /** * Test that relative base href's are resolved correctly: