HER-1737 excessive job.log WARNINGs for misreported encoding (FLV reported as text/html;charset=UTF-8

* ReplayCharSequence.java
    new interface method: getDecoderExceptionCount(), reporting >0 if any decoding problems were encountered and silently replaced
* GenericReplayCharSequence.java
    count decoding problems, and only log as FINE rather than WARNING
* InMemoryReplayCharSequence.java
    try decoding with reporting; if any problems note at least one then decode without reporting errors
* ExtractorHTML.java
    after using ReplayCharSequence, log WARNING if nonzero decoding errors

commons/src/main/java/org/archive/io/GenericReplayCharSequence.java

@@ -92,6 +92,9 @@ public class GenericReplayCharSequence implements ReplayCharSequence {
      */
     protected int length;
     
+    /** counter of decoding exceptions for report at end */
+    protected long decodingExceptions = 0; 
+    
     /**
      * Byte offset into the file where the memory mapped portion begins.
      */
@@ -316,7 +319,8 @@ public class GenericReplayCharSequence implements ReplayCharSequence {
             CharBuffer cbuf = decoder.decode(tempBuf);
             return cbuf.get();
         } catch (CharacterCodingException e) {
-            logger.warning("unable to get character at index=" + index + " (fileIndex=" + fileIndex + "): " + e);
+            logger.log(Level.FINE,"unable to get character at index=" + index + " (fileIndex=" + fileIndex + "): " + e, e);
+            decodingExceptions++; 
             // U+FFFD REPLACEMENT CHARACTER --
             // "used to replace an incoming character whose value is unknown or unrepresentable in Unicode"
             return (char) 0xfffd;
@@ -392,4 +396,12 @@ public class GenericReplayCharSequence implements ReplayCharSequence {
     public int length() {
         return length;
     }
+
+    /* (non-Javadoc)
+     * @see org.archive.io.ReplayCharSequence#getDecodeExceptionCount()
+     */
+    @Override
+    public long getDecodeExceptionCount() {
+        return decodingExceptions;
+    }
 }

commons/src/main/java/org/archive/io/InMemoryReplayCharSequence.java

@@ -22,7 +22,9 @@ package org.archive.io;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.CharBuffer;
+import java.nio.charset.CharacterCodingException;
 import java.nio.charset.Charset;
+import java.nio.charset.CodingErrorAction;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -38,6 +40,8 @@ public class InMemoryReplayCharSequence implements ReplayCharSequence {
      */
     private CharBuffer charBuffer = null;
 
+    protected long decodingExceptionsCount = 0;
+    
     /**
      * Constructor for all in-memory operation.
      *
@@ -93,7 +97,15 @@ public class InMemoryReplayCharSequence implements ReplayCharSequence {
             // TODO: better detection or default
             charset = Charset.forName(FALLBACK_CHARSET_NAME);
         }
-        return charset.decode(bb).asReadOnlyBuffer();
+        try {
+            return charset.newDecoder()
+                   .onMalformedInput(CodingErrorAction.REPORT)
+                   .onUnmappableCharacter(CodingErrorAction.REPORT)
+                   .decode(bb).asReadOnlyBuffer();
+        } catch (CharacterCodingException cce) {
+            decodingExceptionsCount++;
+            return charset.decode(bb).asReadOnlyBuffer();
+        }
     }
 
     public void close() {
@@ -123,4 +135,14 @@ public class InMemoryReplayCharSequence implements ReplayCharSequence {
         sb.append(this);
         return sb.toString();
     }
+    
+    /**
+     * Return 1 if there were decoding problems (a full count isn't possible).
+     * 
+     * @see org.archive.io.ReplayCharSequence#getDecodeExceptionCount()
+     */
+    @Override
+    public long getDecodeExceptionCount() {
+        return decodingExceptionsCount;
+    }
 }

commons/src/main/java/org/archive/io/ReplayCharSequence.java

@@ -46,4 +46,12 @@ public interface ReplayCharSequence extends CharSequence, Closeable {
      * @throws IOException Problem cleaning up file system resources.
      */
     public void close() throws IOException;
+    
+    /**
+     * Report count of decoder errors silently eaten during ReplayCharSequence
+     * use. May be less than the number of individual decoding anomalies in 
+     * underlying content (if decoding method doesn't allow counting individual
+     * errors). 
+     */
+    public long getDecodeExceptionCount(); 
 }

modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java

@@ -653,6 +653,13 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
            cs = curi.getRecorder().getReplayCharSequence();
            // Extract all links from the charsequence
            extract(curi, cs);
+           if(cs.getDecodeExceptionCount()>0) {
+               logger.log(Level.WARNING,
+                       "decoding problem: "+ cs.getDecodeExceptionCount()
+                       + " errors (or more) using "
+                       + curi.getRecorder().getCharacterEncoding() 
+                       + " on CrawlURI "+curi.getURI());
+           }
            // Set flag to indicate that link extraction is completed.
            return true;
         } catch (IOException e) {