feat: strip invalid chars from xml rest api output

engine/src/main/java/org/archive/crawler/restlet/XmlMarshaller.java

@@ -196,7 +196,7 @@ public class XmlMarshaller {
         } else if (marshalAsElement(value)) {
             marshalBean(xmlWriter, key, value);
         } else {
-            xmlWriter.dataElement(key, value.toString());
+            xmlWriter.dataElement(key, stripInvalidXMLChars(value.toString()));
         }
     }
 
@@ -226,4 +226,24 @@ public class XmlMarshaller {
             marshal(xmlWriter, "value", item);
         }
     }
+
+    public static String stripInvalidXMLChars(String input) {
+        if (input == null || input.isEmpty()) {
+            return input;
+        }
+        StringBuilder sb = new StringBuilder();
+        input.codePoints().forEach(cp -> {
+            if (isValidXMLCodePoint(cp)) {
+                sb.appendCodePoint(cp);
+            }
+        });
+        return sb.toString();
+    }
+
+    protected static boolean isValidXMLCodePoint(int cp) {
+        return (cp == 0x9) || (cp == 0xA) || (cp == 0xD) ||
+                (cp >= 0x20 && cp <= 0xD7FF) ||
+                (cp >= 0xE000 && cp <= 0xFFFD) ||
+                (cp >= 0x10000 && cp <= 0x10FFFF);
+    }
 }

engine/src/test/java/org/archive/crawler/restlet/XmlMarshallerTest.java

@@ -1,7 +1,9 @@
 package org.archive.crawler.restlet;
 
 import java.io.StringWriter;
+import java.util.ArrayList;
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 
 import jakarta.xml.bind.annotation.XmlRootElement;
@@ -101,4 +103,35 @@ public class XmlMarshallerTest {
         System.out.println(xml);
         assertTrue(xml.matches(expected), "xml matches expected RE");
     }
+
+    /**
+     * List with invalid character simulating bad input from job.log or crawl.log
+     */
+    @Test
+    public void testInvalidCharacters() throws Exception {
+        List<String> list = new ArrayList<String>();
+        StringBuilder sb = new StringBuilder();
+        sb.append("TEST<>&");
+        sb.append('\u0007'); // BEL - should be excluded
+        sb.append('\uD7FF'); // last code point before surrogates - should be accepted
+        sb.append('\uD800'); // high surrogates - should be excluded
+        sb.append('\uE000'); // first after surrogates - should be accepted
+        sb.append('\uFFFD'); // replacement char - should be accepted
+
+        sb.append(Character.toChars(0x10000)); // lowest supplementary char - should be accepted
+        sb.append(Character.toChars(0x10FFFF)); // highest supplementary char - should be accepted
+
+        list.add(sb.toString());
+        Map<String, Object> map = new LinkedHashMap<String, Object>();
+        map.put("crawlLogTail", list);
+        StringWriter w = new StringWriter();
+        XmlMarshaller.marshalDocument(w, "doc", list);
+
+        String expected = "(?s:)" +
+                "^<\\?xml version=\"1\\.0\" standalone='yes'\\?>\\s*" +
+                "<doc>\\s*<value>TEST&lt;&gt;&amp;&#55295;&#57344;&#65533;&#55296;&#56320;&#56319;&#57343;</value>\\s*" +
+                "</doc>\\s*$";
+        String xml = w.toString();
+        assertTrue(xml.matches(expected), "xml matches expected RE");
+    }
 }