diff --git a/engine/src/main/java/org/archive/crawler/restlet/XmlMarshaller.java b/engine/src/main/java/org/archive/crawler/restlet/XmlMarshaller.java index 4234f732..f22bf46a 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/XmlMarshaller.java +++ b/engine/src/main/java/org/archive/crawler/restlet/XmlMarshaller.java @@ -196,7 +196,7 @@ public class XmlMarshaller { } else if (marshalAsElement(value)) { marshalBean(xmlWriter, key, value); } else { - xmlWriter.dataElement(key, value.toString()); + xmlWriter.dataElement(key, stripInvalidXMLChars(value.toString())); } } @@ -226,4 +226,24 @@ public class XmlMarshaller { marshal(xmlWriter, "value", item); } } + + public static String stripInvalidXMLChars(String input) { + if (input == null || input.isEmpty()) { + return input; + } + StringBuilder sb = new StringBuilder(); + input.codePoints().forEach(cp -> { + if (isValidXMLCodePoint(cp)) { + sb.appendCodePoint(cp); + } + }); + return sb.toString(); + } + + protected static boolean isValidXMLCodePoint(int cp) { + return (cp == 0x9) || (cp == 0xA) || (cp == 0xD) || + (cp >= 0x20 && cp <= 0xD7FF) || + (cp >= 0xE000 && cp <= 0xFFFD) || + (cp >= 0x10000 && cp <= 0x10FFFF); + } } diff --git a/engine/src/test/java/org/archive/crawler/restlet/XmlMarshallerTest.java b/engine/src/test/java/org/archive/crawler/restlet/XmlMarshallerTest.java index 11da6165..a4884304 100644 --- a/engine/src/test/java/org/archive/crawler/restlet/XmlMarshallerTest.java +++ b/engine/src/test/java/org/archive/crawler/restlet/XmlMarshallerTest.java @@ -1,7 +1,9 @@ package org.archive.crawler.restlet; import java.io.StringWriter; +import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import jakarta.xml.bind.annotation.XmlRootElement; @@ -101,4 +103,35 @@ public class XmlMarshallerTest { System.out.println(xml); assertTrue(xml.matches(expected), "xml matches expected RE"); } + + /** + * List with invalid character simulating bad input from job.log or crawl.log + */ + @Test + public void testInvalidCharacters() throws Exception { + List list = new ArrayList(); + StringBuilder sb = new StringBuilder(); + sb.append("TEST<>&"); + sb.append('\u0007'); // BEL - should be excluded + sb.append('\uD7FF'); // last code point before surrogates - should be accepted + sb.append('\uD800'); // high surrogates - should be excluded + sb.append('\uE000'); // first after surrogates - should be accepted + sb.append('\uFFFD'); // replacement char - should be accepted + + sb.append(Character.toChars(0x10000)); // lowest supplementary char - should be accepted + sb.append(Character.toChars(0x10FFFF)); // highest supplementary char - should be accepted + + list.add(sb.toString()); + Map map = new LinkedHashMap(); + map.put("crawlLogTail", list); + StringWriter w = new StringWriter(); + XmlMarshaller.marshalDocument(w, "doc", list); + + String expected = "(?s:)" + + "^<\\?xml version=\"1\\.0\" standalone='yes'\\?>\\s*" + + "\\s*TEST<>&퟿�����\\s*" + + "\\s*$"; + String xml = w.toString(); + assertTrue(xml.matches(expected), "xml matches expected RE"); + } }