From 45b205061fbfe255dc8cae56e316e851ff9f4a68 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Sun, 31 Mar 2013 18:54:12 -0700 Subject: [PATCH 1/4] Fix for testCompressedARCStream - if arc has gzip header but doesn't have IA flags, its still compressed --- .../src/main/java/org/archive/io/arc/ARCReaderFactory.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/commons/src/main/java/org/archive/io/arc/ARCReaderFactory.java b/commons/src/main/java/org/archive/io/arc/ARCReaderFactory.java index d3e65f66..e7dc1625 100644 --- a/commons/src/main/java/org/archive/io/arc/ARCReaderFactory.java +++ b/commons/src/main/java/org/archive/io/arc/ARCReaderFactory.java @@ -285,7 +285,11 @@ implements ARCConstants { compressedARCFile = true; } } + } else { + // Some old arcs don't have an extra header at all, but they're still compressed + compressedARCFile = true; } + return compressedARCFile; } From 8a40f9ff13eacc6ebe906620da29b52597a5cf3d Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Mon, 1 Apr 2013 13:04:33 -0700 Subject: [PATCH 2/4] fix misleading logging --- commons/src/main/java/org/archive/io/WriterPool.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commons/src/main/java/org/archive/io/WriterPool.java b/commons/src/main/java/org/archive/io/WriterPool.java index 79dad027..11931516 100644 --- a/commons/src/main/java/org/archive/io/WriterPool.java +++ b/commons/src/main/java/org/archive/io/WriterPool.java @@ -92,7 +92,7 @@ public abstract class WriterPool { final int poolMaximumActive, final int poolMaximumWait) { logger.info("Initial configuration:" + " prefix=" + settings.getPrefix() + - ", suffix=" + settings.getTemplate() + + ", template=" + settings.getTemplate() + ", compress=" + settings.getCompress() + ", maxSize=" + settings.getMaxFileSizeBytes() + ", maxActive=" + poolMaximumActive + From 4dd46e6af89a1de47540b47a7ff3f9f72dfc3695 Mon Sep 17 00:00:00 2001 From: Kenji Nagahashi Date: Mon, 1 Apr 2013 23:45:22 -0700 Subject: [PATCH 3/4] quick fix to ClassCastException in ToePool.shortReportLineTo(). fixed a typo in Job.ftl. --- .../archive/crawler/framework/ToePool.java | 48 ++++++++++--------- .../org/archive/crawler/restlet/Job.ftl | 2 +- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/framework/ToePool.java b/engine/src/main/java/org/archive/crawler/framework/ToePool.java index 8097337d..808553f4 100644 --- a/engine/src/main/java/org/archive/crawler/framework/ToePool.java +++ b/engine/src/main/java/org/archive/crawler/framework/ToePool.java @@ -283,30 +283,32 @@ public class ToePool extends ThreadGroup implements Reporter { w.print(map.get("toeCount")); w.print(" threads: "); - TreeSet> sortedSteps = (TreeSet>) map.get("steps"); - if(sortedSteps.size()==0) { - return; + LinkedList sortedSteps = (LinkedList)map.get("steps"); + { + Iterator iter = sortedSteps.iterator(); + if (!iter.hasNext()) { + return; + } + w.print(iter.next()); + if (iter.hasNext()) { + w.print(", "); + w.print(iter.next()); + if (iter.hasNext()) { + w.print(", etc..."); + } + } + w.print("; "); } - w.print(Histotable.entryString(sortedSteps.first())); - if(sortedSteps.size()>1) { - Iterator> iter = sortedSteps.iterator(); - iter.next(); - w.print(", "); - w.print(Histotable.entryString(iter.next())); - } - if(sortedSteps.size()>2) { - w.print(", etc..."); - } - w.print("; "); - TreeSet> sortedProcessors = (TreeSet>) map.get("processors"); - w.print(Histotable.entryString(sortedProcessors.first())); - if(sortedProcessors.size()>1) { - Iterator> iter = sortedProcessors.iterator(); - iter.next(); - while(iter.hasNext()) { - w.print(", "); - w.print(Histotable.entryString(iter.next())); - } + LinkedList sortedProcesses = (LinkedList)map.get("processors"); + { + Iterator iter = sortedProcesses.iterator(); + if (iter.hasNext()) { + w.print(iter.next()); + while (iter.hasNext()) { + w.print(", "); + w.print(iter.next()); + } + } } } diff --git a/engine/src/main/resources/org/archive/crawler/restlet/Job.ftl b/engine/src/main/resources/org/archive/crawler/restlet/Job.ftl index 63be2bd1..3ca34f60 100644 --- a/engine/src/main/resources/org/archive/crawler/restlet/Job.ftl +++ b/engine/src/main/resources/org/archive/crawler/restlet/Job.ftl @@ -164,7 +164,7 @@ ${line?html}

Files

-

Browser Job Directory

+

Browse Job Directory

Configuration-referenced Paths

<#assign configRefPaths=job.configFiles! /> <#if !configRefPaths?has_content > From c98c1d0e29ae20e6d26c4df8c96fc4ea9f3f55b1 Mon Sep 17 00:00:00 2001 From: Kenji Nagahashi Date: Tue, 2 Apr 2013 17:55:03 -0700 Subject: [PATCH 4/4] changed CrawlerLoggerModule.setupLogs() from private to protected so that sub-class can modify logging configuration. --- .../java/org/archive/crawler/reporting/CrawlerLoggerModule.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java b/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java index a3d0da54..c635e580 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java +++ b/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java @@ -235,7 +235,7 @@ public class CrawlerLoggerModule closeLogFiles(); } - private void setupLogs() throws IOException { + protected void setupLogs() throws IOException { String logsPath = getPath().getFile().getAbsolutePath() + File.separatorChar; uriProcessing = Logger.getLogger(LOGNAME_CRAWL + "." + logsPath); runtimeErrors = Logger.getLogger(LOGNAME_RUNTIME_ERRORS + "." +