From c1b8138d4d1d103ef15cfebbad5c9120fd2297e9 Mon Sep 17 00:00:00 2001 From: gojomo Date: Mon, 6 Dec 2010 19:47:49 +0000 Subject: [PATCH] [HER-1762] H3 performance tuning Charset lookup sync de-bottlenecking * org/apache/commons/httpclient/EncodingUtil.java use explicit Charset instances or cached to minimize lookups * LaxURLCodec, BdbMultipleWorkQueues use explicit Charset instances --- .../commons/httpclient/util/EncodingUtil.java | 54 ++++++++++++++----- .../java/org/archive/net/LaxURLCodec.java | 4 +- .../frontier/BdbMultipleWorkQueues.java | 10 ++-- 3 files changed, 46 insertions(+), 22 deletions(-) diff --git a/commons/src/main/java/org/apache/commons/httpclient/util/EncodingUtil.java b/commons/src/main/java/org/apache/commons/httpclient/util/EncodingUtil.java index 31ae3495..6c57c1de 100644 --- a/commons/src/main/java/org/apache/commons/httpclient/util/EncodingUtil.java +++ b/commons/src/main/java/org/apache/commons/httpclient/util/EncodingUtil.java @@ -30,6 +30,8 @@ package org.apache.commons.httpclient.util; import java.io.UnsupportedEncodingException; +import java.nio.charset.Charset; // IA CHANGE +import java.util.concurrent.ConcurrentMap; // IA CHANGE import org.apache.commons.codec.net.URLCodec; import org.apache.commons.httpclient.HttpClientError; @@ -37,6 +39,11 @@ import org.apache.commons.httpclient.NameValuePair; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import com.google.common.base.Charsets; // IA CHANGE +import com.google.common.base.Function; // IA CHANGE +import com.google.common.collect.ComputationException; // IA CHANGE +import com.google.common.collect.MapMaker; // IA CHANGE + /** * The home for utility methods that handle various encoding tasks. * @@ -53,6 +60,15 @@ public class EncodingUtil { /** Log object for this class. */ private static final Log LOG = LogFactory.getLog(EncodingUtil.class); +// BEGIN IA CHANGES + private static final ConcurrentMap cachedCharsets = + new MapMaker().concurrencyLevel(64).makeComputingMap(new Function() { + @Override + public Charset apply(String charsetName) { + return Charset.forName(charsetName); + } + }); +// END IA CHANGES /** * Form-urlencoding routine. * @@ -160,15 +176,18 @@ public class EncodingUtil { throw new IllegalArgumentException("charset may not be null or empty"); } +// BEGIN IA CHANGES try { - return new String(data, offset, length, charset); - } catch (UnsupportedEncodingException e) { + Charset cs = cachedCharsets.get(charset); + return new String(data, offset, length, cs); + } catch (ComputationException e) { if (LOG.isWarnEnabled()) { LOG.warn("Unsupported encoding: " + charset + ". System encoding used"); } return new String(data, offset, length); } +// END IA CHANGES } @@ -207,9 +226,11 @@ public class EncodingUtil { throw new IllegalArgumentException("charset may not be null or empty"); } +// BEGIN IA CHANGES try { - return data.getBytes(charset); - } catch (UnsupportedEncodingException e) { + Charset cs = cachedCharsets.get(charset); + return data.getBytes(cs); + } catch (ComputationException e) { if (LOG.isWarnEnabled()) { LOG.warn("Unsupported encoding: " + charset + ". System encoding used."); @@ -217,6 +238,7 @@ public class EncodingUtil { return data.getBytes(); } +// END IA CHANGES } /** @@ -233,11 +255,13 @@ public class EncodingUtil { throw new IllegalArgumentException("Parameter may not be null"); } - try { - return data.getBytes("US-ASCII"); - } catch (UnsupportedEncodingException e) { - throw new HttpClientError("HttpClient requires ASCII support"); - } +// BEGIN IA CHANGES +// try { + return data.getBytes(Charsets.US_ASCII); +// } catch (UnsupportedEncodingException e) { +// throw new HttpClientError("HttpClient requires ASCII support"); +// } +// END IA CHANGES } /** @@ -258,11 +282,13 @@ public class EncodingUtil { throw new IllegalArgumentException("Parameter may not be null"); } - try { - return new String(data, offset, length, "US-ASCII"); - } catch (UnsupportedEncodingException e) { - throw new HttpClientError("HttpClient requires ASCII support"); - } +// BEGIN IA CHANGES +// try { + return new String(data, offset, length, Charsets.US_ASCII); +// } catch (UnsupportedEncodingException e) { +// throw new HttpClientError("HttpClient requires ASCII support"); +// } +// END IA CHANGES } /** diff --git a/commons/src/main/java/org/archive/net/LaxURLCodec.java b/commons/src/main/java/org/archive/net/LaxURLCodec.java index f60ff3f2..ef2567bc 100644 --- a/commons/src/main/java/org/archive/net/LaxURLCodec.java +++ b/commons/src/main/java/org/archive/net/LaxURLCodec.java @@ -24,6 +24,8 @@ import java.util.BitSet; import org.apache.commons.codec.net.URLCodec; +import com.google.common.base.Charsets; + /** * @author gojomo */ @@ -153,6 +155,6 @@ public class LaxURLCodec extends URLCodec { if (pString == null) { return null; } - return new String(encodeUrl(safe,pString.getBytes(cs)), "US-ASCII"); + return new String(encodeUrl(safe,pString.getBytes(cs)), Charsets.US_ASCII); } } diff --git a/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java b/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java index ac5c8a03..957ec3cb 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java +++ b/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java @@ -35,6 +35,7 @@ import org.archive.bdb.KryoBinding; import org.archive.modules.CrawlURI; import org.archive.util.ArchiveUtils; +import com.google.common.base.Charsets; import com.sleepycat.bind.EntryBinding; import com.sleepycat.bind.serial.StoredClassCatalog; import com.sleepycat.je.Cursor; @@ -424,13 +425,8 @@ public class BdbMultipleWorkQueues { static DatabaseEntry calculateInsertKey(CrawlURI curi) { byte[] classKeyBytes = null; int len = 0; - try { - classKeyBytes = curi.getClassKey().getBytes("UTF-8"); - len = classKeyBytes.length; - } catch (UnsupportedEncodingException e) { - // should be impossible; all JVMs must support UTF-8 - e.printStackTrace(); - } + classKeyBytes = curi.getClassKey().getBytes(Charsets.UTF_8); + len = classKeyBytes.length; byte[] keyData = new byte[len+9]; System.arraycopy(classKeyBytes,0,keyData,0,len); keyData[len]=0;