calmer logging (fewer alerts in job.log/alerts.log/heritrix_out.log)

* Recorder
    object to unsupported content-encodings when first set
* FetchHTTP
    note as annotation unsupported content-encodings
* Link
    not as annotation when base URI is used for absent via
* ExtractorHTML
    downgrade char-sequence reading problem (often a chunking problem) to WARNING from SEVERE

commons/src/main/java/org/archive/util/Recorder.java

@@ -25,6 +25,8 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.nio.charset.Charset;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.zip.DeflaterInputStream;
@@ -325,11 +327,23 @@ public class Recorder {
         this.inputIsChunked = chunked;
     }
     
+    static Set<String> SUPPORTED_ENCODINGS = new HashSet<String>();
+    static {
+        SUPPORTED_ENCODINGS.add("gzip"); 
+        SUPPORTED_ENCODINGS.add("x-gzip");
+        SUPPORTED_ENCODINGS.add("deflate");
+        SUPPORTED_ENCODINGS.add("identity");
+        SUPPORTED_ENCODINGS.add("none"); // unofficial but common
+    }
     /**
      * @param contentEncoding declared content-encoding of input recording.
      */
     public void setContentEncoding(String contentEncoding) {
-        this.contentEncoding = contentEncoding;
+        String lowerCoding = contentEncoding.toLowerCase(); 
+        if(!SUPPORTED_ENCODINGS.contains(contentEncoding.toLowerCase())) {
+            throw new IllegalArgumentException("contentEncoding unsupported: "+contentEncoding); 
+        }
+        this.contentEncoding = lowerCoding;
     }
 
     /**
@@ -466,10 +480,11 @@ public class Recorder {
             }
         } else if ("deflate".equalsIgnoreCase(contentEncoding)) {
             return new DeflaterInputStream(entityStream);
-        } else if ("identity".equalsIgnoreCase(contentEncoding)) {
+        } else if ("identity".equalsIgnoreCase(contentEncoding) || "none".equalsIgnoreCase(contentEncoding)) {
             return entityStream;
         } else {
-            logger.log(Level.WARNING,"Unknown content-encoding '"+contentEncoding+"' declared; using raw entity instead");
+            // shouldn't be reached given check on setContentEncoding
+            logger.log(Level.INFO,"Unknown content-encoding '"+contentEncoding+"' declared; using raw entity instead");
             return entityStream; 
         }
     }

modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java

@@ -697,7 +697,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
            return true;
         } catch (IOException e) {
             curi.getNonFatalFailures().add(e);
-            logger.log(Level.SEVERE,"Failed get of replay char sequence in " +
+            logger.log(Level.WARNING,"Failed get of replay char sequence in " +
                 Thread.currentThread().getName(), e);
         }
         return false;

modules/src/main/java/org/archive/modules/extractor/Link.java

@@ -137,7 +137,8 @@ public class Link implements Serializable, Comparable<Link> {
             String newUri, LinkContext context, Hop hop) throws URIException {
         UURI relTo = uri.getVia();
         if(relTo==null) {
-            LOGGER.warning("no via where expected; using base instead: "+uri);
+            LOGGER.info("no via where expected; using base instead: "+uri);
+            uri.getAnnotations().add("usedBaseForVia");
             relTo = uri.getBaseURI();
         }
         UURI dest = UURIFactory.getInstance(relTo, newUri);

modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java

@@ -818,7 +818,11 @@ public class FetchHTTP extends Processor implements Lifecycle {
         Header contentEncodingHeader = ((HttpMethodBase) method).getResponseHeader("Content-Encoding"); 
         if (contentEncodingHeader!=null) {
             String ce = contentEncodingHeader.getValue().trim(); 
-            rec.setContentEncoding(ce); 
+            try {
+                rec.setContentEncoding(ce); 
+            } catch (IllegalArgumentException e) {
+                uri.getAnnotations().add("unsatisfiableContentEncoding:"+StringUtils.stripToEmpty(ce));
+            }
         }
     }