From 6a2716ce0035c2a91e58a97aea64a20831f40aca Mon Sep 17 00:00:00 2001 From: gojomo Date: Fri, 12 Nov 2010 19:37:52 +0000 Subject: [PATCH] Reduce synchronization bottlenecks * FrontierJournal, CrawlerJournal do barest minimum inside 'synchronized' method --- .../src/main/java/org/archive/io/CrawlerJournal.java | 7 +------ .../org/archive/crawler/frontier/FrontierJournal.java | 11 ++--------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/commons/src/main/java/org/archive/io/CrawlerJournal.java b/commons/src/main/java/org/archive/io/CrawlerJournal.java index 5f98a6a9..420afb14 100644 --- a/commons/src/main/java/org/archive/io/CrawlerJournal.java +++ b/commons/src/main/java/org/archive/io/CrawlerJournal.java @@ -92,15 +92,10 @@ public class CrawlerJournal implements Closeable { this.out = initialize(gzipFile); } - /** - * Allocate a buffer for accumulating lines to write and reuse it. - */ - protected MutableString accumulatingBuffer = new MutableString(1024); - protected Writer initialize(final File f) throws FileNotFoundException, IOException { FileUtils.moveAsideIfExists(f); return new OutputStreamWriter(new GZIPOutputStream( - new FastBufferedOutputStream(new FileOutputStream(f)))); + new FastBufferedOutputStream(new FileOutputStream(f),32*1024))); } /** diff --git a/engine/src/main/java/org/archive/crawler/frontier/FrontierJournal.java b/engine/src/main/java/org/archive/crawler/frontier/FrontierJournal.java index 54dc6ca6..764bd8d5 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/FrontierJournal.java +++ b/engine/src/main/java/org/archive/crawler/frontier/FrontierJournal.java @@ -84,15 +84,8 @@ public class FrontierJournal extends CrawlerJournal { writeLongUriLine(F_ADD, curi); } - public synchronized void writeLongUriLine(String tag, CrawlURI curi) { - accumulatingBuffer.length(0); - this.accumulatingBuffer.append(tag). - append(curi.toString()). - append(" "). - append(curi.getPathFromSeed()). - append(" "). - append(curi.flattenVia()); - writeLine(accumulatingBuffer); + public void writeLongUriLine(String tag, CrawlURI curi) { + writeLine(tag, curi.toString(), " ",curi.getPathFromSeed(), " ", curi.flattenVia()); } public void finishedSuccess(CrawlURI curi) {