diff --git a/commons/src/main/java/org/archive/io/GenerationFileHandler.java b/commons/src/main/java/org/archive/io/GenerationFileHandler.java index 03e21ad5..21c3d12e 100644 --- a/commons/src/main/java/org/archive/io/GenerationFileHandler.java +++ b/commons/src/main/java/org/archive/io/GenerationFileHandler.java @@ -24,8 +24,10 @@ import java.io.IOException; import java.util.LinkedList; import java.util.List; import java.util.logging.FileHandler; +import java.util.logging.Formatter; import java.util.logging.LogRecord; +import org.archive.crawler.io.Preformatter; import org.archive.util.FileUtils; @@ -138,10 +140,19 @@ public class GenerationFileHandler extends FileHandler { @Override public void publish(LogRecord record) { - // preformat outside synchronized superclass method - // (our most involved UriProcessingFormatter will cache result for LogRecord) - getFormatter().format(record); - super.publish(record); + // when possible preformat outside synchronized superclass method + // (our most involved UriProcessingFormatter can cache result) + Formatter f = getFormatter(); + if(!(f instanceof Preformatter)) { + super.publish(record); + } else { + try { + ((Preformatter)f).preformat(record); + super.publish(record); + } finally { + ((Preformatter)f).clear(); + } + } } // // TODO: determine if there's another way to have this optimization without diff --git a/engine/src/main/java/org/archive/crawler/io/Preformatter.java b/engine/src/main/java/org/archive/crawler/io/Preformatter.java new file mode 100644 index 00000000..b0a3c596 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/io/Preformatter.java @@ -0,0 +1,32 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.crawler.io; + +import java.util.logging.LogRecord; + +/** + * Interface indicating a logging Formatter can preformat a record (outside + * the standard-implementation synchronized block) and cache it, returning it + * for the next request for formatting from the same thread. + * @contributor gojomo + */ +public interface Preformatter { + public void preformat(LogRecord record); + public void clear(); +} diff --git a/engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java b/engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java index 6be14cd4..fc0ac5d8 100644 --- a/engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java +++ b/engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java @@ -34,7 +34,7 @@ import org.archive.util.MimetypeUtils; * @author gojomo */ public class UriProcessingFormatter -extends Formatter implements CoreAttributeConstants { +extends Formatter implements Preformatter, CoreAttributeConstants { private final static String NA = "-"; /** * Guess at line length (URIs are assumed avg. of 128 bytes). @@ -57,7 +57,6 @@ extends Formatter implements CoreAttributeConstants { } }; - protected final ThreadLocal cachedRecord = new ThreadLocal(); protected final ThreadLocal cachedFormat = new ThreadLocal(); protected boolean logExtraInfo; @@ -66,7 +65,7 @@ extends Formatter implements CoreAttributeConstants { } public String format(LogRecord lr) { - if(lr==cachedRecord.get()) { + if(cachedFormat.get()!=null) { return cachedFormat.get(); } CrawlURI curi = (CrawlURI)lr.getParameters()[0]; @@ -153,10 +152,7 @@ extends Formatter implements CoreAttributeConstants { } buffer.append("\n"); - cachedRecord.set(lr); - String formatted = buffer.toString(); - cachedFormat.set(formatted); - return formatted; + return buffer.toString(); } /** @@ -166,6 +162,16 @@ extends Formatter implements CoreAttributeConstants { protected String checkForNull(String str) { return (str == null || str.length() <= 0)? NA: str; } + + @Override + public void clear() { + cachedFormat.set(null); + } + + @Override + public void preformat(LogRecord record) { + cachedFormat.set(format(record)); + } } diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index f8ded234..41a1d878 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -876,6 +876,12 @@ implements MultiReporter, Serializable, OverlayContext { this.linkExtractorFinished = false; // Clean the data map of all but registered permanent members. this.data = getPersistentDataMap(); + + extraInfo = null; + overlayNames = null; + outCandidates = null; + outLinks = null; + method = null; } public Map getPersistentDataMap() {