Fix for HER-2032 - test stream for compression; add tests with and without compression

commons/src/main/java/org/archive/io/arc/ARCReaderFactory.java

@@ -19,6 +19,7 @@
 
 package org.archive.io.arc;
 
+import java.io.BufferedInputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
@@ -132,9 +133,26 @@ implements ARCConstants {
     protected ArchiveReader getArchiveReader(final String arc,
 			final InputStream is, final boolean atFirstRecord)
 			throws IOException {
-		// For now, assume stream is compressed. Later add test of input
-		// stream or handle exception thrown when figure not compressed stream.
-		return new CompressedARCReader(arc, is, atFirstRecord);
+
+        // We do this mark() reset() stuff, wrapping in a BufferedInputStream if
+        // necessary to make it work, because testCompressedARCStream() consumes
+        // some bytes from the input stream
+        InputStream possiblyWrapped;
+        if (is.markSupported()) {
+            possiblyWrapped = is;
+        } else {
+            possiblyWrapped = new BufferedInputStream(is);
+        }
+
+        possiblyWrapped.mark(100);
+        boolean compressed = testCompressedARCStream(possiblyWrapped);
+        possiblyWrapped.reset();
+
+        if (compressed) {
+            return new CompressedARCReader(arc, possiblyWrapped, atFirstRecord);
+        } else {
+            return new UncompressedARCReader(arc, possiblyWrapped);
+        }
 	}
     
     /**
@@ -248,8 +266,8 @@ implements ARCConstants {
         GzipHeader gh = null;
         try {
             gh = new GzipHeader(is);
-        } catch (NoGzipMagicException e ) {
-            return compressedARCFile;
+        } catch (NoGzipMagicException e) {
+            return false;
         }
         
         byte[] fextra = gh.getFextra();
@@ -311,7 +329,7 @@ implements ARCConstants {
         public UncompressedARCReader(final String f, final InputStream is) {
             // Arc file has been tested for existence by time it has come
             // to here.
-            setIn(is);
+            setIn(new CountingInputStream(is));
             initialize(f);
         }
     }

commons/src/test/java/org/archive/io/arc/ARCReaderFactoryTest.java

@@ -20,11 +20,14 @@
 package org.archive.io.arc;
 
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.Iterator;
 
+import org.archive.io.ArchiveReader;
 import org.archive.io.ArchiveRecord;
 import org.archive.util.TmpDirTestCase;
 
@@ -94,4 +97,33 @@ public class ARCReaderFactoryTest extends TmpDirTestCase {
         reader.close();
         doGetFileUrl(arc);
     }   
+    
+    public void testGetCompressedArcStream() throws IOException {
+        testGetArcStream(true);
+    }
+    
+    public void testGetUncompressedArcStream() throws IOException {
+        testGetArcStream(false);
+    }
+
+    protected void testGetArcStream(boolean compress) throws IOException, FileNotFoundException {
+        File arc = ARCWriterTest.createARCFile(getTmpDir(), true);
+        ArchiveReader reader = ARCReaderFactory.get(null, new FileInputStream(arc), compress);
+        assertNotNull(reader);
+        Iterator<ArchiveRecord> i = reader.iterator();
+        
+        // ARC header
+        assertTrue(i.hasNext());
+        ARCRecord r = (ARCRecord)i.next();
+        assertEquals("filedesc://test.arc", r.getHeader().getHeaderValue("subject-uri"));
+        
+        // 1 fake http record
+        assertTrue(i.hasNext());
+        r = (ARCRecord)i.next();
+        assertEquals(200, r.getStatusCode());
+        assertEquals("http://www.archive.org/test/", r.getHeader().getHeaderValue("subject-uri"));
+        
+        assertFalse(i.hasNext());
+        reader.close();
+    }
 }