lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-26 07:35:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Minimize transient garbage creation identified by allocation profiling

Browse Source 
* UURIFactory, canonicalize/**, PathologicalPathDecideRule, ExtractorHTML
    use recycled matchers via TextUtils
* CrawlURI, KeyedProperties, OverlayContext, SheetOverlaysManager
    use ArrayList and indexed access rather than LinkedList and iterator instances
This commit is contained in:
gojomo
2010-11-05 21:34:53 +00:00
parent acedca1788
commit f4d69b37da
12 changed files with 64 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/src/main/java/org/archive/net/UURIFactory.java
+9 -3
View File
@@ -372,17 +372,20 @@ public class UURIFactory extends URI {
// Test for the case of more than two slashes after the http(s) scheme.
// Replace with two slashes as mozilla does if found.
// See [ 788219 ] URI Syntax Errors stop page parsing.
Matcher matcher = HTTP_SCHEME_SLASHES.matcher(uri);
// Matcher matcher = HTTP_SCHEME_SLASHES.matcher(uri);
Matcher matcher = TextUtils.getMatcher(HTTP_SCHEME_SLASHES.pattern(), uri);
if (matcher.matches()) {
uri = matcher.group(1) + matcher.group(2);
}
TextUtils.recycleMatcher(matcher);
// now, minimally escape any whitespace
uri = escapeWhitespace(uri);
// For further processing, get uri elements. See the RFC2396REGEX
// comment above for explanation of group indices used in the below.
matcher = RFC2396REGEX.matcher(uri);
// matcher = RFC2396REGEX.matcher(uri);
matcher = TextUtils.getMatcher(RFC2396REGEX.pattern(), uri);
if (!matcher.matches()) {
throw new URIException("Failed parse of " + uri);
}
@@ -392,6 +395,7 @@ public class UURIFactory extends URI {
String uriPath = checkUriElement(matcher.group(6));
String uriQuery = checkUriElement(matcher.group(8));
// UNUSED String uriFragment = checkUriElement(matcher.group(10));
TextUtils.recycleMatcher(matcher); matcher = null;
// Test if relative URI. If so, need a base to resolve against.
if (uriScheme == null || uriScheme.length() <= 0) {
@@ -701,7 +705,8 @@ public class UURIFactory extends URI {
*/
private String checkPort(String uriAuthority)
throws URIException {
Matcher m = PORTREGEX.matcher(uriAuthority);
// Matcher m = PORTREGEX.matcher(uriAuthority);
Matcher m = TextUtils.getMatcher(PORTREGEX.pattern(), uriAuthority);
if (m.matches()) {
String no = m.group(2);
if (no != null && no.length() > 0) {
@@ -726,6 +731,7 @@ public class UURIFactory extends URI {
}
}
}
TextUtils.recycleMatcher(m);
return uriAuthority;
}
commons/src/main/java/org/archive/spring/KeyedProperties.java
+17 -16
View File
@@ -21,7 +21,6 @@
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -56,19 +55,21 @@ public class KeyedProperties extends ConcurrentHashMap<String,Object> {
* @return discovered override, or local value
*/
public Object get(String key) {
if(!threadOverrides.get().isEmpty()) {
for(OverlayContext ocontext: threadOverrides.get()) {
for(String name: ocontext.getOverlayNames()) {
Map<String,Object> m = ocontext.getOverlayMap(name);
for(String ok : getOverrideKeys(key)) {
Object val = m.get(ok);
if(val!=null) {
return val;
}
ArrayList<OverlayContext> overlays = threadOverrides.get();
for(int i = overlays.size()-1; i>=0; i--) {
OverlayContext ocontext = overlays.get(i);
for(int j = ocontext.getOverlayNames().size()-1; j>=0; j--) {
String name = ocontext.getOverlayNames().get(j);
Map<String,Object> m = ocontext.getOverlayMap(name);
for(String ok : getOverrideKeys(key)) {
Object val = m.get(ok);
if(val!=null) {
return val;
}
}
}
}
return super.get(key);
}
@@ -95,10 +96,10 @@ public class KeyedProperties extends ConcurrentHashMap<String,Object> {
/**
* ThreadLocal (contextual) collection of pushed override maps
*/
static ThreadLocal<LinkedList<OverlayContext>> threadOverrides =
new ThreadLocal<LinkedList<OverlayContext>>() {
protected LinkedList<OverlayContext> initialValue() {
return new LinkedList<OverlayContext>();
static ThreadLocal<ArrayList<OverlayContext>> threadOverrides =
new ThreadLocal<ArrayList<OverlayContext>>() {
protected ArrayList<OverlayContext> initialValue() {
return new ArrayList<OverlayContext>();
}
};
/**
@@ -106,7 +107,7 @@ public class KeyedProperties extends ConcurrentHashMap<String,Object> {
* @param m Map to add
*/
static public void pushOverrideContext(OverlayContext ocontext) {
threadOverrides.get().addFirst(ocontext);
threadOverrides.get().add(ocontext);
}
/**
@@ -115,7 +116,7 @@ public class KeyedProperties extends ConcurrentHashMap<String,Object> {
*/
static public OverlayContext popOverridesContext() {
// TODO maybe check that pop is as expected
return threadOverrides.get().removeFirst();
return threadOverrides.get().remove(threadOverrides.get().size()-1);
}
static public void clearAllOverrideContexts() {
commons/src/main/java/org/archive/spring/OverlayContext.java
+2 -2
View File
@@ -19,7 +19,7 @@
package org.archive.spring;
import java.util.LinkedList;
import java.util.ArrayList;
import java.util.Map;
/**
@@ -32,7 +32,7 @@ public interface OverlayContext {
* (even if in fact no overlays were added) */
public boolean haveOverlayNamesBeenSet();
/** return a list of the names of overlay maps to consider */
LinkedList<String> getOverlayNames();
ArrayList<String> getOverlayNames();
/** get the map corresponding to the overlay name */
Map<String,Object> getOverlayMap(String name);
}
engine/src/main/java/org/archive/crawler/spring/SheetOverlaysManager.java
+1 -1
View File
@@ -315,7 +315,7 @@ BeanFactoryAware, OverlayMapsSource, ApplicationListener {
List<String> foundPrefixes = PrefixFinder.findKeys(sheetNamesBySurt, effectiveSurt);
for(String prefix : foundPrefixes) {
for(String name : sheetNamesBySurt.get(prefix)) {
curi.getOverlayNames().push(name);
curi.getOverlayNames().add(name);
}
}
// apply deciderule-based overlays
modules/src/main/java/org/archive/modules/canonicalize/BaseRule.java
+7 -2
View File
@@ -23,6 +23,7 @@ import java.util.regex.Matcher;
import org.archive.spring.HasKeyedProperties;
import org.archive.spring.KeyedProperties;
import org.archive.util.TextUtils;
/**
* Base of all rules applied canonicalizing a URL that are configurable
@@ -36,6 +37,7 @@ import org.archive.spring.KeyedProperties;
*/
public abstract class BaseRule
implements CanonicalizationRule, Serializable, HasKeyedProperties {
private static final long serialVersionUID = 1L;
protected KeyedProperties kp = new KeyedProperties();
public KeyedProperties getKeyedProperties() {
return kp;
@@ -69,10 +71,13 @@ implements CanonicalizationRule, Serializable, HasKeyedProperties {
* @return Original <code>url</code> else concatenization of group 1
* and group 2.
*/
protected String doStripRegexMatch(String url, Matcher matcher) {
return (matcher != null && matcher.matches())?
protected String doStripRegexMatch(String url, String pat) {
Matcher matcher = TextUtils.getMatcher(pat, url);
String retVal = (matcher != null && matcher.matches())?
checkForNull(matcher.group(1)) + checkForNull(matcher.group(2)):
url;
TextUtils.recycleMatcher(matcher);
return retVal;
}
/**
modules/src/main/java/org/archive/modules/canonicalize/StripSessionCFIDs.java
+1 -1
View File
@@ -58,7 +58,7 @@ extends BaseRule {
}
public String canonicalize(String url) {
return doStripRegexMatch(url, COLDFUSION_PATTERN.matcher(url));
return doStripRegexMatch(url, COLDFUSION_PATTERN.pattern());
}
}
modules/src/main/java/org/archive/modules/canonicalize/StripSessionIDs.java
+3 -3
View File
@@ -70,9 +70,9 @@ extends BaseRule {
}
public String canonicalize(String url) {
url = doStripRegexMatch(url, BASE_PATTERN.matcher(url));
url = doStripRegexMatch(url, SID_PATTERN.matcher(url));
url = doStripRegexMatch(url, ASPSESSION_PATTERN.matcher(url));
url = doStripRegexMatch(url, BASE_PATTERN.pattern());
url = doStripRegexMatch(url, SID_PATTERN.pattern());
url = doStripRegexMatch(url, ASPSESSION_PATTERN.pattern());
return url;
}
}
modules/src/main/java/org/archive/modules/canonicalize/StripUserinfoRule.java
+1 -1
View File
@@ -46,6 +46,6 @@ public class StripUserinfoRule extends BaseRule {
}
public String canonicalize(String url) {
return doStripRegexMatch(url, REGEX.matcher(url));
return doStripRegexMatch(url, REGEX.pattern());
}
}
modules/src/main/java/org/archive/modules/canonicalize/StripWWWNRule.java
+1 -1
View File
@@ -53,6 +53,6 @@ public class StripWWWNRule extends BaseRule {
}
public String canonicalize(String url) {
return doStripRegexMatch(url, REGEX.matcher(url));
return doStripRegexMatch(url, REGEX.pattern());
}
}
modules/src/main/java/org/archive/modules/canonicalize/StripWWWRule.java
+1 -1
View File
@@ -49,6 +49,6 @@ public class StripWWWRule extends BaseRule {
}
public String canonicalize(String url) {
return doStripRegexMatch(url, REGEX.matcher(url));
return doStripRegexMatch(url, REGEX.pattern());
}
}
modules/src/main/java/org/archive/modules/deciderules/PathologicalPathDecideRule.java
+13 -27
View File
@@ -18,10 +18,10 @@
*/
package org.archive.modules.deciderules;
import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
import org.archive.modules.CrawlURI;
import org.archive.util.TextUtils;
/**
@@ -48,8 +48,6 @@ public class PathologicalPathDecideRule extends DecideRule {
public void setMaxRepetitions(int maxRepetitions) {
kp.put("maxRepetitions", maxRepetitions);
}
private AtomicReference<Pattern> pattern = new AtomicReference<Pattern>();
/** Constructs a new PathologicalPathFilter.
*
@@ -62,31 +60,19 @@ public class PathologicalPathDecideRule extends DecideRule {
@Override
protected DecideResult innerDecide(CrawlURI uri) {
int maxRep = getMaxRepetitions();
Pattern p = getPattern(maxRep);
if (p.matcher(uri.getUURI().toString()).matches()) {
return DecideResult.REJECT;
} else {
return DecideResult.NONE;
// Pattern p = getPattern(maxRep);
Matcher m = TextUtils.getMatcher(constructRegex(maxRep), uri.getUURI().toString());
try {
if (m.matches()) {
return DecideResult.REJECT;
} else {
return DecideResult.NONE;
}
} finally {
TextUtils.recycleMatcher(m);
}
}
/**
* Construct the regex string to be matched against the URI.
* @param o an object to extract a URI from.
* @return the regex pattern.
*/
private Pattern getPattern(int maxRep) {
// race no concern: assignment is atomic, happy with any last value
Pattern p = pattern.get();
if (p != null) {
return p;
}
String regex = constructRegex(maxRep);
p = Pattern.compile(regex);
pattern.set(p);
return p;
}
protected String constructRegex(int rep) {
return (rep == 0) ? null : ".*?/(.*?/)\\1{" + rep + ",}.*";
}
modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java
+8 -8
View File
@@ -27,7 +27,6 @@ import java.util.Iterator;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.httpclient.URIException;
import org.archive.io.ReplayCharSequence;
@@ -297,8 +296,9 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
this.metadata = provider;
}
private Pattern relevantTagExtractor;
private Pattern eachAttributeExtractor;
// TODO: convert to Strings
private String relevantTagPattern;
private String eachAttributePattern;
public ExtractorHTML() {
}
@@ -307,21 +307,21 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
String regex = RELEVANT_TAG_EXTRACTOR;
regex = regex.replace(MAX_ELEMENT_REPLACE,
Integer.toString(getMaxElementLength()));
this.relevantTagExtractor = Pattern.compile(regex);
this.relevantTagPattern = regex;
regex = EACH_ATTRIBUTE_EXTRACTOR;
regex = regex.replace(MAX_ATTR_NAME_REPLACE,
Integer.toString(getMaxAttributeNameLength()));
regex = regex.replace(MAX_ATTR_VAL_REPLACE,
Integer.toString(getMaxAttributeValLength()));
this.eachAttributeExtractor = Pattern.compile(regex);
this.eachAttributePattern = regex;
}
protected void processGeneralTag(CrawlURI curi, CharSequence element,
CharSequence cs) {
Matcher attr = eachAttributeExtractor.matcher(cs);
Matcher attr = TextUtils.getMatcher(eachAttributePattern,cs);
// Just in case it's an OBJECT or APPLET tag
String codebase = null;
@@ -687,7 +687,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
* of this extractors' lifetime.
*/
void extract(CrawlURI curi, CharSequence cs) {
Matcher tags = relevantTagExtractor.matcher(cs);
Matcher tags = TextUtils.getMatcher(relevantTagPattern,cs);
while(tags.find()) {
if(Thread.interrupted()){
break;
@@ -801,7 +801,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
* @return True robots exclusion metatag.
*/
protected boolean processMeta(CrawlURI curi, CharSequence cs) {
Matcher attr = eachAttributeExtractor.matcher(cs);
Matcher attr = TextUtils.getMatcher(eachAttributePattern,cs);
String name = null;
String httpEquiv = null;
String content = null;