From f4d69b37da1c913c25e2a0b0ffeeda9dd2bd7528 Mon Sep 17 00:00:00 2001 From: gojomo Date: Fri, 5 Nov 2010 21:34:53 +0000 Subject: [PATCH] Minimize transient garbage creation identified by allocation profiling * UURIFactory, canonicalize/**, PathologicalPathDecideRule, ExtractorHTML use recycled matchers via TextUtils * CrawlURI, KeyedProperties, OverlayContext, SheetOverlaysManager use ArrayList and indexed access rather than LinkedList and iterator instances --- .../java/org/archive/net/UURIFactory.java | 12 ++++-- .../org/archive/spring/KeyedProperties.java | 33 +++++++-------- .../org/archive/spring/OverlayContext.java | 4 +- .../crawler/spring/SheetOverlaysManager.java | 2 +- .../modules/canonicalize/BaseRule.java | 9 ++++- .../canonicalize/StripSessionCFIDs.java | 2 +- .../modules/canonicalize/StripSessionIDs.java | 6 +-- .../canonicalize/StripUserinfoRule.java | 2 +- .../modules/canonicalize/StripWWWNRule.java | 2 +- .../modules/canonicalize/StripWWWRule.java | 2 +- .../PathologicalPathDecideRule.java | 40 ++++++------------- .../modules/extractor/ExtractorHTML.java | 16 ++++---- 12 files changed, 64 insertions(+), 66 deletions(-) diff --git a/commons/src/main/java/org/archive/net/UURIFactory.java b/commons/src/main/java/org/archive/net/UURIFactory.java index 1c29b036..62cb79e3 100644 --- a/commons/src/main/java/org/archive/net/UURIFactory.java +++ b/commons/src/main/java/org/archive/net/UURIFactory.java @@ -372,17 +372,20 @@ public class UURIFactory extends URI { // Test for the case of more than two slashes after the http(s) scheme. // Replace with two slashes as mozilla does if found. // See [ 788219 ] URI Syntax Errors stop page parsing. - Matcher matcher = HTTP_SCHEME_SLASHES.matcher(uri); +// Matcher matcher = HTTP_SCHEME_SLASHES.matcher(uri); + Matcher matcher = TextUtils.getMatcher(HTTP_SCHEME_SLASHES.pattern(), uri); if (matcher.matches()) { uri = matcher.group(1) + matcher.group(2); } + TextUtils.recycleMatcher(matcher); // now, minimally escape any whitespace uri = escapeWhitespace(uri); // For further processing, get uri elements. See the RFC2396REGEX // comment above for explanation of group indices used in the below. - matcher = RFC2396REGEX.matcher(uri); +// matcher = RFC2396REGEX.matcher(uri); + matcher = TextUtils.getMatcher(RFC2396REGEX.pattern(), uri); if (!matcher.matches()) { throw new URIException("Failed parse of " + uri); } @@ -392,6 +395,7 @@ public class UURIFactory extends URI { String uriPath = checkUriElement(matcher.group(6)); String uriQuery = checkUriElement(matcher.group(8)); // UNUSED String uriFragment = checkUriElement(matcher.group(10)); + TextUtils.recycleMatcher(matcher); matcher = null; // Test if relative URI. If so, need a base to resolve against. if (uriScheme == null || uriScheme.length() <= 0) { @@ -701,7 +705,8 @@ public class UURIFactory extends URI { */ private String checkPort(String uriAuthority) throws URIException { - Matcher m = PORTREGEX.matcher(uriAuthority); +// Matcher m = PORTREGEX.matcher(uriAuthority); + Matcher m = TextUtils.getMatcher(PORTREGEX.pattern(), uriAuthority); if (m.matches()) { String no = m.group(2); if (no != null && no.length() > 0) { @@ -726,6 +731,7 @@ public class UURIFactory extends URI { } } } + TextUtils.recycleMatcher(m); return uriAuthority; } diff --git a/commons/src/main/java/org/archive/spring/KeyedProperties.java b/commons/src/main/java/org/archive/spring/KeyedProperties.java index c6ea55e3..a6e5adbf 100644 --- a/commons/src/main/java/org/archive/spring/KeyedProperties.java +++ b/commons/src/main/java/org/archive/spring/KeyedProperties.java @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.HashSet; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -56,19 +55,21 @@ public class KeyedProperties extends ConcurrentHashMap { * @return discovered override, or local value */ public Object get(String key) { - if(!threadOverrides.get().isEmpty()) { - for(OverlayContext ocontext: threadOverrides.get()) { - for(String name: ocontext.getOverlayNames()) { - Map m = ocontext.getOverlayMap(name); - for(String ok : getOverrideKeys(key)) { - Object val = m.get(ok); - if(val!=null) { - return val; - } + ArrayList overlays = threadOverrides.get(); + for(int i = overlays.size()-1; i>=0; i--) { + OverlayContext ocontext = overlays.get(i); + for(int j = ocontext.getOverlayNames().size()-1; j>=0; j--) { + String name = ocontext.getOverlayNames().get(j); + Map m = ocontext.getOverlayMap(name); + for(String ok : getOverrideKeys(key)) { + Object val = m.get(ok); + if(val!=null) { + return val; } } } } + return super.get(key); } @@ -95,10 +96,10 @@ public class KeyedProperties extends ConcurrentHashMap { /** * ThreadLocal (contextual) collection of pushed override maps */ - static ThreadLocal> threadOverrides = - new ThreadLocal>() { - protected LinkedList initialValue() { - return new LinkedList(); + static ThreadLocal> threadOverrides = + new ThreadLocal>() { + protected ArrayList initialValue() { + return new ArrayList(); } }; /** @@ -106,7 +107,7 @@ public class KeyedProperties extends ConcurrentHashMap { * @param m Map to add */ static public void pushOverrideContext(OverlayContext ocontext) { - threadOverrides.get().addFirst(ocontext); + threadOverrides.get().add(ocontext); } /** @@ -115,7 +116,7 @@ public class KeyedProperties extends ConcurrentHashMap { */ static public OverlayContext popOverridesContext() { // TODO maybe check that pop is as expected - return threadOverrides.get().removeFirst(); + return threadOverrides.get().remove(threadOverrides.get().size()-1); } static public void clearAllOverrideContexts() { diff --git a/commons/src/main/java/org/archive/spring/OverlayContext.java b/commons/src/main/java/org/archive/spring/OverlayContext.java index ebccd0ab..ddd90b0f 100644 --- a/commons/src/main/java/org/archive/spring/OverlayContext.java +++ b/commons/src/main/java/org/archive/spring/OverlayContext.java @@ -19,7 +19,7 @@ package org.archive.spring; -import java.util.LinkedList; +import java.util.ArrayList; import java.util.Map; /** @@ -32,7 +32,7 @@ public interface OverlayContext { * (even if in fact no overlays were added) */ public boolean haveOverlayNamesBeenSet(); /** return a list of the names of overlay maps to consider */ - LinkedList getOverlayNames(); + ArrayList getOverlayNames(); /** get the map corresponding to the overlay name */ Map getOverlayMap(String name); } diff --git a/engine/src/main/java/org/archive/crawler/spring/SheetOverlaysManager.java b/engine/src/main/java/org/archive/crawler/spring/SheetOverlaysManager.java index 0517fb65..276f17a6 100644 --- a/engine/src/main/java/org/archive/crawler/spring/SheetOverlaysManager.java +++ b/engine/src/main/java/org/archive/crawler/spring/SheetOverlaysManager.java @@ -315,7 +315,7 @@ BeanFactoryAware, OverlayMapsSource, ApplicationListener { List foundPrefixes = PrefixFinder.findKeys(sheetNamesBySurt, effectiveSurt); for(String prefix : foundPrefixes) { for(String name : sheetNamesBySurt.get(prefix)) { - curi.getOverlayNames().push(name); + curi.getOverlayNames().add(name); } } // apply deciderule-based overlays diff --git a/modules/src/main/java/org/archive/modules/canonicalize/BaseRule.java b/modules/src/main/java/org/archive/modules/canonicalize/BaseRule.java index ebb08b72..f7e14c94 100644 --- a/modules/src/main/java/org/archive/modules/canonicalize/BaseRule.java +++ b/modules/src/main/java/org/archive/modules/canonicalize/BaseRule.java @@ -23,6 +23,7 @@ import java.util.regex.Matcher; import org.archive.spring.HasKeyedProperties; import org.archive.spring.KeyedProperties; +import org.archive.util.TextUtils; /** * Base of all rules applied canonicalizing a URL that are configurable @@ -36,6 +37,7 @@ import org.archive.spring.KeyedProperties; */ public abstract class BaseRule implements CanonicalizationRule, Serializable, HasKeyedProperties { + private static final long serialVersionUID = 1L; protected KeyedProperties kp = new KeyedProperties(); public KeyedProperties getKeyedProperties() { return kp; @@ -69,10 +71,13 @@ implements CanonicalizationRule, Serializable, HasKeyedProperties { * @return Original url else concatenization of group 1 * and group 2. */ - protected String doStripRegexMatch(String url, Matcher matcher) { - return (matcher != null && matcher.matches())? + protected String doStripRegexMatch(String url, String pat) { + Matcher matcher = TextUtils.getMatcher(pat, url); + String retVal = (matcher != null && matcher.matches())? checkForNull(matcher.group(1)) + checkForNull(matcher.group(2)): url; + TextUtils.recycleMatcher(matcher); + return retVal; } /** diff --git a/modules/src/main/java/org/archive/modules/canonicalize/StripSessionCFIDs.java b/modules/src/main/java/org/archive/modules/canonicalize/StripSessionCFIDs.java index 3eef8fbd..de5456ef 100644 --- a/modules/src/main/java/org/archive/modules/canonicalize/StripSessionCFIDs.java +++ b/modules/src/main/java/org/archive/modules/canonicalize/StripSessionCFIDs.java @@ -58,7 +58,7 @@ extends BaseRule { } public String canonicalize(String url) { - return doStripRegexMatch(url, COLDFUSION_PATTERN.matcher(url)); + return doStripRegexMatch(url, COLDFUSION_PATTERN.pattern()); } } \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/canonicalize/StripSessionIDs.java b/modules/src/main/java/org/archive/modules/canonicalize/StripSessionIDs.java index b7473217..93a8a506 100644 --- a/modules/src/main/java/org/archive/modules/canonicalize/StripSessionIDs.java +++ b/modules/src/main/java/org/archive/modules/canonicalize/StripSessionIDs.java @@ -70,9 +70,9 @@ extends BaseRule { } public String canonicalize(String url) { - url = doStripRegexMatch(url, BASE_PATTERN.matcher(url)); - url = doStripRegexMatch(url, SID_PATTERN.matcher(url)); - url = doStripRegexMatch(url, ASPSESSION_PATTERN.matcher(url)); + url = doStripRegexMatch(url, BASE_PATTERN.pattern()); + url = doStripRegexMatch(url, SID_PATTERN.pattern()); + url = doStripRegexMatch(url, ASPSESSION_PATTERN.pattern()); return url; } } \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/canonicalize/StripUserinfoRule.java b/modules/src/main/java/org/archive/modules/canonicalize/StripUserinfoRule.java index 35045e79..c0392eb2 100644 --- a/modules/src/main/java/org/archive/modules/canonicalize/StripUserinfoRule.java +++ b/modules/src/main/java/org/archive/modules/canonicalize/StripUserinfoRule.java @@ -46,6 +46,6 @@ public class StripUserinfoRule extends BaseRule { } public String canonicalize(String url) { - return doStripRegexMatch(url, REGEX.matcher(url)); + return doStripRegexMatch(url, REGEX.pattern()); } } diff --git a/modules/src/main/java/org/archive/modules/canonicalize/StripWWWNRule.java b/modules/src/main/java/org/archive/modules/canonicalize/StripWWWNRule.java index de38fc74..a10607e7 100644 --- a/modules/src/main/java/org/archive/modules/canonicalize/StripWWWNRule.java +++ b/modules/src/main/java/org/archive/modules/canonicalize/StripWWWNRule.java @@ -53,6 +53,6 @@ public class StripWWWNRule extends BaseRule { } public String canonicalize(String url) { - return doStripRegexMatch(url, REGEX.matcher(url)); + return doStripRegexMatch(url, REGEX.pattern()); } } \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/canonicalize/StripWWWRule.java b/modules/src/main/java/org/archive/modules/canonicalize/StripWWWRule.java index 317dee6f..cd985ff5 100644 --- a/modules/src/main/java/org/archive/modules/canonicalize/StripWWWRule.java +++ b/modules/src/main/java/org/archive/modules/canonicalize/StripWWWRule.java @@ -49,6 +49,6 @@ public class StripWWWRule extends BaseRule { } public String canonicalize(String url) { - return doStripRegexMatch(url, REGEX.matcher(url)); + return doStripRegexMatch(url, REGEX.pattern()); } } diff --git a/modules/src/main/java/org/archive/modules/deciderules/PathologicalPathDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/PathologicalPathDecideRule.java index 4da1557d..593e4747 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/PathologicalPathDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/PathologicalPathDecideRule.java @@ -18,10 +18,10 @@ */ package org.archive.modules.deciderules; -import java.util.concurrent.atomic.AtomicReference; -import java.util.regex.Pattern; +import java.util.regex.Matcher; import org.archive.modules.CrawlURI; +import org.archive.util.TextUtils; /** @@ -48,8 +48,6 @@ public class PathologicalPathDecideRule extends DecideRule { public void setMaxRepetitions(int maxRepetitions) { kp.put("maxRepetitions", maxRepetitions); } - - private AtomicReference pattern = new AtomicReference(); /** Constructs a new PathologicalPathFilter. * @@ -62,31 +60,19 @@ public class PathologicalPathDecideRule extends DecideRule { @Override protected DecideResult innerDecide(CrawlURI uri) { int maxRep = getMaxRepetitions(); - Pattern p = getPattern(maxRep); - if (p.matcher(uri.getUURI().toString()).matches()) { - return DecideResult.REJECT; - } else { - return DecideResult.NONE; +// Pattern p = getPattern(maxRep); + Matcher m = TextUtils.getMatcher(constructRegex(maxRep), uri.getUURI().toString()); + try { + if (m.matches()) { + return DecideResult.REJECT; + } else { + return DecideResult.NONE; + } + } finally { + TextUtils.recycleMatcher(m); } } - - /** - * Construct the regex string to be matched against the URI. - * @param o an object to extract a URI from. - * @return the regex pattern. - */ - private Pattern getPattern(int maxRep) { - // race no concern: assignment is atomic, happy with any last value - Pattern p = pattern.get(); - if (p != null) { - return p; - } - String regex = constructRegex(maxRep); - p = Pattern.compile(regex); - pattern.set(p); - return p; - } - + protected String constructRegex(int rep) { return (rep == 0) ? null : ".*?/(.*?/)\\1{" + rep + ",}.*"; } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index b704a85f..feb44f0d 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -27,7 +27,6 @@ import java.util.Iterator; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.apache.commons.httpclient.URIException; import org.archive.io.ReplayCharSequence; @@ -297,8 +296,9 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean this.metadata = provider; } - private Pattern relevantTagExtractor; - private Pattern eachAttributeExtractor; + // TODO: convert to Strings + private String relevantTagPattern; + private String eachAttributePattern; public ExtractorHTML() { } @@ -307,21 +307,21 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean String regex = RELEVANT_TAG_EXTRACTOR; regex = regex.replace(MAX_ELEMENT_REPLACE, Integer.toString(getMaxElementLength())); - this.relevantTagExtractor = Pattern.compile(regex); + this.relevantTagPattern = regex; regex = EACH_ATTRIBUTE_EXTRACTOR; regex = regex.replace(MAX_ATTR_NAME_REPLACE, Integer.toString(getMaxAttributeNameLength())); regex = regex.replace(MAX_ATTR_VAL_REPLACE, Integer.toString(getMaxAttributeValLength())); - this.eachAttributeExtractor = Pattern.compile(regex); + this.eachAttributePattern = regex; } protected void processGeneralTag(CrawlURI curi, CharSequence element, CharSequence cs) { - Matcher attr = eachAttributeExtractor.matcher(cs); + Matcher attr = TextUtils.getMatcher(eachAttributePattern,cs); // Just in case it's an OBJECT or APPLET tag String codebase = null; @@ -687,7 +687,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean * of this extractors' lifetime. */ void extract(CrawlURI curi, CharSequence cs) { - Matcher tags = relevantTagExtractor.matcher(cs); + Matcher tags = TextUtils.getMatcher(relevantTagPattern,cs); while(tags.find()) { if(Thread.interrupted()){ break; @@ -801,7 +801,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean * @return True robots exclusion metatag. */ protected boolean processMeta(CrawlURI curi, CharSequence cs) { - Matcher attr = eachAttributeExtractor.matcher(cs); + Matcher attr = TextUtils.getMatcher(eachAttributePattern,cs); String name = null; String httpEquiv = null; String content = null;