diff --git a/commons/src/main/java/org/archive/util/Recorder.java b/commons/src/main/java/org/archive/util/Recorder.java index 4875a698..61167d4f 100644 --- a/commons/src/main/java/org/archive/util/Recorder.java +++ b/commons/src/main/java/org/archive/util/Recorder.java @@ -25,6 +25,8 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.nio.charset.Charset; +import java.util.HashSet; +import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; import java.util.zip.DeflaterInputStream; @@ -325,11 +327,23 @@ public class Recorder { this.inputIsChunked = chunked; } + static Set SUPPORTED_ENCODINGS = new HashSet(); + static { + SUPPORTED_ENCODINGS.add("gzip"); + SUPPORTED_ENCODINGS.add("x-gzip"); + SUPPORTED_ENCODINGS.add("deflate"); + SUPPORTED_ENCODINGS.add("identity"); + SUPPORTED_ENCODINGS.add("none"); // unofficial but common + } /** * @param contentEncoding declared content-encoding of input recording. */ public void setContentEncoding(String contentEncoding) { - this.contentEncoding = contentEncoding; + String lowerCoding = contentEncoding.toLowerCase(); + if(!SUPPORTED_ENCODINGS.contains(contentEncoding.toLowerCase())) { + throw new IllegalArgumentException("contentEncoding unsupported: "+contentEncoding); + } + this.contentEncoding = lowerCoding; } /** @@ -466,10 +480,11 @@ public class Recorder { } } else if ("deflate".equalsIgnoreCase(contentEncoding)) { return new DeflaterInputStream(entityStream); - } else if ("identity".equalsIgnoreCase(contentEncoding)) { + } else if ("identity".equalsIgnoreCase(contentEncoding) || "none".equalsIgnoreCase(contentEncoding)) { return entityStream; } else { - logger.log(Level.WARNING,"Unknown content-encoding '"+contentEncoding+"' declared; using raw entity instead"); + // shouldn't be reached given check on setContentEncoding + logger.log(Level.INFO,"Unknown content-encoding '"+contentEncoding+"' declared; using raw entity instead"); return entityStream; } } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index c90b2797..351ad90e 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -697,7 +697,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean return true; } catch (IOException e) { curi.getNonFatalFailures().add(e); - logger.log(Level.SEVERE,"Failed get of replay char sequence in " + + logger.log(Level.WARNING,"Failed get of replay char sequence in " + Thread.currentThread().getName(), e); } return false; diff --git a/modules/src/main/java/org/archive/modules/extractor/Link.java b/modules/src/main/java/org/archive/modules/extractor/Link.java index 0c155f46..15897251 100644 --- a/modules/src/main/java/org/archive/modules/extractor/Link.java +++ b/modules/src/main/java/org/archive/modules/extractor/Link.java @@ -137,7 +137,8 @@ public class Link implements Serializable, Comparable { String newUri, LinkContext context, Hop hop) throws URIException { UURI relTo = uri.getVia(); if(relTo==null) { - LOGGER.warning("no via where expected; using base instead: "+uri); + LOGGER.info("no via where expected; using base instead: "+uri); + uri.getAnnotations().add("usedBaseForVia"); relTo = uri.getBaseURI(); } UURI dest = UURIFactory.getInstance(relTo, newUri); diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index a0b282db..55a68e66 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -818,7 +818,11 @@ public class FetchHTTP extends Processor implements Lifecycle { Header contentEncodingHeader = ((HttpMethodBase) method).getResponseHeader("Content-Encoding"); if (contentEncodingHeader!=null) { String ce = contentEncodingHeader.getValue().trim(); - rec.setContentEncoding(ce); + try { + rec.setContentEncoding(ce); + } catch (IllegalArgumentException e) { + uri.getAnnotations().add("unsatisfiableContentEncoding:"+StringUtils.stripToEmpty(ce)); + } } }