diff --git a/commons/src/main/java/org/archive/io/GenericReplayCharSequence.java b/commons/src/main/java/org/archive/io/GenericReplayCharSequence.java index 11161010..236d4e75 100644 --- a/commons/src/main/java/org/archive/io/GenericReplayCharSequence.java +++ b/commons/src/main/java/org/archive/io/GenericReplayCharSequence.java @@ -92,6 +92,9 @@ public class GenericReplayCharSequence implements ReplayCharSequence { */ protected int length; + /** counter of decoding exceptions for report at end */ + protected long decodingExceptions = 0; + /** * Byte offset into the file where the memory mapped portion begins. */ @@ -316,7 +319,8 @@ public class GenericReplayCharSequence implements ReplayCharSequence { CharBuffer cbuf = decoder.decode(tempBuf); return cbuf.get(); } catch (CharacterCodingException e) { - logger.warning("unable to get character at index=" + index + " (fileIndex=" + fileIndex + "): " + e); + logger.log(Level.FINE,"unable to get character at index=" + index + " (fileIndex=" + fileIndex + "): " + e, e); + decodingExceptions++; // U+FFFD REPLACEMENT CHARACTER -- // "used to replace an incoming character whose value is unknown or unrepresentable in Unicode" return (char) 0xfffd; @@ -392,4 +396,12 @@ public class GenericReplayCharSequence implements ReplayCharSequence { public int length() { return length; } + + /* (non-Javadoc) + * @see org.archive.io.ReplayCharSequence#getDecodeExceptionCount() + */ + @Override + public long getDecodeExceptionCount() { + return decodingExceptions; + } } \ No newline at end of file diff --git a/commons/src/main/java/org/archive/io/InMemoryReplayCharSequence.java b/commons/src/main/java/org/archive/io/InMemoryReplayCharSequence.java index 3c6cb233..dc6c5ce0 100644 --- a/commons/src/main/java/org/archive/io/InMemoryReplayCharSequence.java +++ b/commons/src/main/java/org/archive/io/InMemoryReplayCharSequence.java @@ -22,7 +22,9 @@ package org.archive.io; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.CharBuffer; +import java.nio.charset.CharacterCodingException; import java.nio.charset.Charset; +import java.nio.charset.CodingErrorAction; import java.util.logging.Level; import java.util.logging.Logger; @@ -38,6 +40,8 @@ public class InMemoryReplayCharSequence implements ReplayCharSequence { */ private CharBuffer charBuffer = null; + protected long decodingExceptionsCount = 0; + /** * Constructor for all in-memory operation. * @@ -93,7 +97,15 @@ public class InMemoryReplayCharSequence implements ReplayCharSequence { // TODO: better detection or default charset = Charset.forName(FALLBACK_CHARSET_NAME); } - return charset.decode(bb).asReadOnlyBuffer(); + try { + return charset.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(bb).asReadOnlyBuffer(); + } catch (CharacterCodingException cce) { + decodingExceptionsCount++; + return charset.decode(bb).asReadOnlyBuffer(); + } } public void close() { @@ -123,4 +135,14 @@ public class InMemoryReplayCharSequence implements ReplayCharSequence { sb.append(this); return sb.toString(); } + + /** + * Return 1 if there were decoding problems (a full count isn't possible). + * + * @see org.archive.io.ReplayCharSequence#getDecodeExceptionCount() + */ + @Override + public long getDecodeExceptionCount() { + return decodingExceptionsCount; + } } diff --git a/commons/src/main/java/org/archive/io/ReplayCharSequence.java b/commons/src/main/java/org/archive/io/ReplayCharSequence.java index 32785136..5e3547ff 100644 --- a/commons/src/main/java/org/archive/io/ReplayCharSequence.java +++ b/commons/src/main/java/org/archive/io/ReplayCharSequence.java @@ -46,4 +46,12 @@ public interface ReplayCharSequence extends CharSequence, Closeable { * @throws IOException Problem cleaning up file system resources. */ public void close() throws IOException; + + /** + * Report count of decoder errors silently eaten during ReplayCharSequence + * use. May be less than the number of individual decoding anomalies in + * underlying content (if decoding method doesn't allow counting individual + * errors). + */ + public long getDecodeExceptionCount(); } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index ff379327..8438b49f 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -653,6 +653,13 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean cs = curi.getRecorder().getReplayCharSequence(); // Extract all links from the charsequence extract(curi, cs); + if(cs.getDecodeExceptionCount()>0) { + logger.log(Level.WARNING, + "decoding problem: "+ cs.getDecodeExceptionCount() + + " errors (or more) using " + + curi.getRecorder().getCharacterEncoding() + + " on CrawlURI "+curi.getURI()); + } // Set flag to indicate that link extraction is completed. return true; } catch (IOException e) {