From 78755dd30425ca21fdfc77134e4087bb0b2c4dc3 Mon Sep 17 00:00:00 2001
From: nlevitt <nlevitt@daa5b2f2-a927-0410-8b2d-f5f262fa301a>
Date: Tue, 14 Jun 2011 02:11:21 +0000
Subject: [PATCH] HER-1898 - Heritrix 3.1 misses many pages that Heritrix 3.0
 catches - reported by Dan Yadgar * GenericReplayCharSequence.java    
 decode() - read another character to check for more content, rather than
 consult BufferedReader.ready(), since the latter sometimes returns false even
 when there is more to read

---
 .../archive/io/GenericReplayCharSequence.java    | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/commons/src/main/java/org/archive/io/GenericReplayCharSequence.java b/commons/src/main/java/org/archive/io/GenericReplayCharSequence.java
index bf49c2b1..1af3922b 100644
--- a/commons/src/main/java/org/archive/io/GenericReplayCharSequence.java
+++ b/commons/src/main/java/org/archive/io/GenericReplayCharSequence.java
@@ -147,8 +147,8 @@ public class GenericReplayCharSequence implements ReplayCharSequence {
                                      String backingFilename,
                                      Charset charset) throws IOException {
         super();
-        logger.fine("new GenericReplayCharSequence() characterEncoding="
-                + charset + " backingFilename=" + backingFilename);
+        logger.fine("characterEncoding=" + charset + " backingFilename="
+                + backingFilename);
 
         if(charset==null) {
             charset = ReplayCharSequence.FALLBACK_CHARSET;
@@ -212,8 +212,8 @@ public class GenericReplayCharSequence implements ReplayCharSequence {
         BufferedReader reader = new BufferedReader(new InputStreamReader(
                 inStream, charset));
 
-        logger.fine("decodeToFile: backingFilename=" + backingFilename
-                + " encoding=" + charset + " decodedFile=" + decodedFile);
+        logger.fine("backingFilename=" + backingFilename + " encoding="
+                + charset + " decodedFile=" + decodedFile);
 
         this.prefixBuffer = CharBuffer.allocate(prefixMax); 
         
@@ -225,8 +225,11 @@ public class GenericReplayCharSequence implements ReplayCharSequence {
             }
             count += read; 
         }
-        
-        if(reader.ready()) {
+
+        int ch = reader.read();
+        if(ch >= 0) {
+            count++;
+            
             // more to decode to file overflow
             this.decodedFile = new File(backingFilename + "." + WRITE_ENCODING);
 
@@ -245,6 +248,7 @@ public class GenericReplayCharSequence implements ReplayCharSequence {
             }
             
             Writer writer = new OutputStreamWriter(fos,WRITE_ENCODING);
+            writer.write(ch);
             count += IOUtils.copyLarge(reader, writer); 
             writer.close();
             reader.close();