diff --git a/commons/src/main/java/org/archive/util/ArchiveUtils.java b/commons/src/main/java/org/archive/util/ArchiveUtils.java index 12bc67ac..614cae76 100644 --- a/commons/src/main/java/org/archive/util/ArchiveUtils.java +++ b/commons/src/main/java/org/archive/util/ArchiveUtils.java @@ -741,22 +741,6 @@ public class ArchiveUtils { pw.flush(); return sw.toString(); } - - /** - * Compose the requested report into a String. DANGEROUS IF REPORT - * CAN BE LARGE. - * - * @param rep Reported - * @param name String name of report to compose - * @return String of report - */ - public static String writeReportToString(MultiReporter rep, String name) { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - rep.reportTo(name,pw); - pw.flush(); - return sw.toString(); - } /** * Enhance given object's default String display for appearing @@ -773,8 +757,8 @@ public class ArchiveUtils { } else if (obj instanceof Map) { return prettyString((Map) obj); } else { - return "<"+obj+">"; - } + return "<"+obj+">"; + } } /** diff --git a/commons/src/main/java/org/archive/util/DevUtils.java b/commons/src/main/java/org/archive/util/DevUtils.java index 98776f1b..d630a0b1 100644 --- a/commons/src/main/java/org/archive/util/DevUtils.java +++ b/commons/src/main/java/org/archive/util/DevUtils.java @@ -55,8 +55,8 @@ public class DevUtils { StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); final Thread current = Thread.currentThread(); - if (current instanceof MultiReporter) { - MultiReporter tt = (MultiReporter)current; + if (current instanceof Reporter) { + Reporter tt = (Reporter)current; try { tt.reportTo(pw); } catch (IOException e) { diff --git a/commons/src/main/java/org/archive/util/MultiReporter.java b/commons/src/main/java/org/archive/util/MultiReporter.java deleted file mode 100644 index 9190ca08..00000000 --- a/commons/src/main/java/org/archive/util/MultiReporter.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.archive.util; - -import java.io.PrintWriter; - -/** - * @contributor stack - */ -public interface MultiReporter extends Reporter { - /** - * Get an array of report names offered by this Reporter. - * A name in brackets indicates a free-form String, - * in accordance with the informal description inside - * the brackets, may yield a useful report. - * - * @return String array of report names, empty if there is only - * one report type - */ - public String[] getReports(); - - /** - * Make a report of the given name to the passed-in Writer, - * If null, give the default report. - * - * @param writer to receive report - */ - public void reportTo(String name, PrintWriter writer); -} diff --git a/commons/src/main/java/org/archive/util/Reporter.java b/commons/src/main/java/org/archive/util/Reporter.java index 72f88df2..2fcb8cd8 100644 --- a/commons/src/main/java/org/archive/util/Reporter.java +++ b/commons/src/main/java/org/archive/util/Reporter.java @@ -37,6 +37,7 @@ public interface Reporter { * * @param writer to receive report */ + @Deprecated public void shortReportLineTo(PrintWriter pw) throws IOException; diff --git a/engine/src/main/java/org/archive/crawler/framework/Frontier.java b/engine/src/main/java/org/archive/crawler/framework/Frontier.java index a33295c8..20eaa389 100644 --- a/engine/src/main/java/org/archive/crawler/framework/Frontier.java +++ b/engine/src/main/java/org/archive/crawler/framework/Frontier.java @@ -28,7 +28,7 @@ import org.archive.modules.CrawlURI; import org.archive.modules.deciderules.DecideRule; import org.archive.modules.fetcher.FetchStats; import org.archive.util.IdentityCacheable; -import org.archive.util.MultiReporter; +import org.archive.util.Reporter; import org.json.JSONException; import org.springframework.context.Lifecycle; @@ -101,7 +101,7 @@ import org.springframework.context.Lifecycle; * @see org.archive.crawler.framework.ToeThread * @see org.archive.crawler.settings.ModuleType */ -public interface Frontier extends Lifecycle, MultiReporter { +public interface Frontier extends Lifecycle, Reporter { /** * Get the next URI that should be processed. If no URI becomes availible diff --git a/engine/src/main/java/org/archive/crawler/framework/ToePool.java b/engine/src/main/java/org/archive/crawler/framework/ToePool.java index 990f0f11..d3cb9a01 100644 --- a/engine/src/main/java/org/archive/crawler/framework/ToePool.java +++ b/engine/src/main/java/org/archive/crawler/framework/ToePool.java @@ -23,14 +23,14 @@ import java.io.PrintWriter; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; -import java.util.TreeSet; import java.util.Map.Entry; +import java.util.TreeSet; import org.apache.commons.lang.StringUtils; import org.archive.crawler.reporting.AlertThreadGroup; import org.archive.util.ArchiveUtils; import org.archive.util.Histotable; -import org.archive.util.MultiReporter; +import org.archive.util.Reporter; /** * A collection of ToeThreads. The class manages the ToeThreads currently @@ -43,7 +43,7 @@ import org.archive.util.MultiReporter; * * @see org.archive.crawler.framework.ToeThread */ -public class ToePool extends ThreadGroup implements MultiReporter { +public class ToePool extends ThreadGroup implements Reporter { /** run worker thread slightly lower than usual */ public static int DEFAULT_TOE_PRIORITY = Thread.NORM_PRIORITY - 1; @@ -70,7 +70,6 @@ public class ToePool extends ThreadGroup implements MultiReporter { toe.interrupt(); } } -// this.controller = null; } /** @@ -190,37 +189,15 @@ public class ToePool extends ThreadGroup implements MultiReporter { // Reporter implementation // - public static String STANDARD_REPORT = "standard"; - public static String COMPACT_REPORT = "compact"; - protected static String[] REPORTS = {STANDARD_REPORT,COMPACT_REPORT}; - - public String[] getReports() { - return REPORTS; - } - - public void reportTo(String name, PrintWriter writer) { - if(COMPACT_REPORT.equals(name)) { - compactReportTo(writer); - return; - } - if(name!=null && !STANDARD_REPORT.equals(name)) { - writer.print(name); - writer.print(" not recognized: giving standard report/n"); - } - standardReportTo(writer); - } - - /* (non-Javadoc) - * @see org.archive.util.Reporter#reportTo(java.io.Writer) - */ - protected void standardReportTo(PrintWriter writer) { + @Override + public void reportTo(PrintWriter writer) { writer.print("Toe threads report - " + ArchiveUtils.get12DigitDate() + "\n"); writer.print(" Job being crawled: " + this.controller.getMetadata().getJobName() + "\n"); writer.print(" Number of toe threads in pool: " + getToeCount() + " (" + getActiveToeCount() + " active)\n\n"); - + Thread[] toes = this.getToes(); synchronized (toes) { for (int i = 0; i < toes.length; i++) { @@ -233,12 +210,9 @@ public class ToePool extends ThreadGroup implements MultiReporter { } } } - } + } - /* (non-Javadoc) - * @see org.archive.util.Reporter#reportTo(java.io.Writer) - */ - protected void compactReportTo(PrintWriter writer) { + public void compactReportTo(PrintWriter writer) { writer.print(getToeCount() + " threads (" + getActiveToeCount() + " active)\n"); @@ -262,6 +236,7 @@ public class ToePool extends ThreadGroup implements MultiReporter { } } + @Override public Map shortReportMap() { Histotable steps = new Histotable(); Histotable processors = new Histotable(); @@ -291,6 +266,7 @@ public class ToePool extends ThreadGroup implements MultiReporter { } @SuppressWarnings("unchecked") + @Override public void shortReportLineTo(PrintWriter w) { Map map = shortReportMap(); w.print(map.get("toeCount")); @@ -327,14 +303,11 @@ public class ToePool extends ThreadGroup implements MultiReporter { /* (non-Javadoc) * @see org.archive.util.Reporter#singleLineLegend() */ + @Override public String shortReportLegend() { return "total: mostCommonStateTotal secondMostCommonStateTotal"; } - public void reportTo(PrintWriter writer) { - reportTo(null,writer); - } - public void waitForAll() { while (true) try { diff --git a/engine/src/main/java/org/archive/crawler/framework/ToeThread.java b/engine/src/main/java/org/archive/crawler/framework/ToeThread.java index e6aefd19..155d95cc 100644 --- a/engine/src/main/java/org/archive/crawler/framework/ToeThread.java +++ b/engine/src/main/java/org/archive/crawler/framework/ToeThread.java @@ -42,9 +42,9 @@ import org.archive.modules.fetcher.HostResolver; import org.archive.spring.KeyedProperties; import org.archive.util.ArchiveUtils; import org.archive.util.DevUtils; -import org.archive.util.MultiReporter; import org.archive.util.ProgressStatisticsReporter; import org.archive.util.Recorder; +import org.archive.util.Reporter; import com.sleepycat.util.RuntimeExceptionWrapper; @@ -55,7 +55,7 @@ import com.sleepycat.util.RuntimeExceptionWrapper; * @author Gordon Mohr */ public class ToeThread extends Thread -implements MultiReporter, ProgressStatisticsReporter, +implements Reporter, ProgressStatisticsReporter, HostResolver, SinkHandlerLogThread, ChainStatusReceiver { public enum Step { @@ -396,10 +396,10 @@ implements MultiReporter, ProgressStatisticsReporter, /** * Compiles and returns a report on its status. - * @param name Report name. * @param pw Where to print. */ - public void reportTo(String name, PrintWriter pw) { + @Override + public void reportTo(PrintWriter pw) { // name is ignored for now: only one kind of report pw.print("["); @@ -488,6 +488,7 @@ implements MultiReporter, ProgressStatisticsReporter, } } + @Override public Map shortReportMap() { Map data = new LinkedHashMap(); data.put("serialNumber", serialNumber); @@ -518,6 +519,7 @@ implements MultiReporter, ProgressStatisticsReporter, /** * @param w PrintWriter to write to. */ + @Override public void shortReportLineTo(PrintWriter w) { w.print("#"); @@ -565,21 +567,10 @@ implements MultiReporter, ProgressStatisticsReporter, w.flush(); } + @Override public String shortReportLegend() { return "#serialNumber processorName currentUri (fetchAttempts) threadState threadStep"; } - - /* (non-Javadoc) - * @see org.archive.util.Reporter#getReports() - */ - public String[] getReports() { - // for now none but the default - return new String[] {}; - } - - public void reportTo(PrintWriter writer) { - reportTo(null, writer); - } public String shortReportLine() { return ArchiveUtils.shortReportLine(this); diff --git a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java index 023cbaa0..c9df326e 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java @@ -39,7 +39,6 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; -import java.io.PrintWriter; import java.util.Collection; import java.util.Iterator; import java.util.concurrent.atomic.AtomicLong; @@ -1131,10 +1130,6 @@ public abstract class AbstractFrontier return ArchiveUtils.shortReportLine(this); } - public void reportTo(PrintWriter writer) { - reportTo(null, writer); - } - public void onApplicationEvent(ApplicationEvent event) { if(event instanceof CrawlStateEvent) { CrawlStateEvent event1 = (CrawlStateEvent)event; diff --git a/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java index 5c2278b7..e72fbf5b 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java @@ -21,9 +21,9 @@ package org.archive.crawler.frontier; import java.io.BufferedReader; import java.io.IOException; import java.io.PrintWriter; +import java.util.Map.Entry; import java.util.Queue; import java.util.SortedMap; -import java.util.Map.Entry; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.DelayQueue; import java.util.concurrent.LinkedBlockingQueue; diff --git a/engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java b/engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java index 5ce4488b..363e085a 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java +++ b/engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java @@ -38,8 +38,8 @@ import org.archive.modules.fetcher.FetchStats; import org.archive.modules.fetcher.FetchStats.Stage; import org.archive.util.ArchiveUtils; import org.archive.util.IdentityCacheable; -import org.archive.util.MultiReporter; import org.archive.util.ObjectIdentityCache; +import org.archive.util.Reporter; /** * A single queue of related URIs to visit, grouped by a classKey @@ -49,7 +49,7 @@ import org.archive.util.ObjectIdentityCache; * @author Christian Kohlschuetter */ public abstract class WorkQueue implements Frontier.FrontierGroup, - Serializable, MultiReporter, Delayed, IdentityCacheable { + Serializable, Reporter, Delayed, IdentityCacheable { private static final long serialVersionUID = -3199666138837266341L; private static final Logger logger = Logger.getLogger(WorkQueue.class.getName()); @@ -451,20 +451,7 @@ public abstract class WorkQueue implements Frontier.FrontierGroup, // Reporter // - /* (non-Javadoc) - * @see org.archive.util.Reporter#getReports() - */ - public String[] getReports() { - return new String[] {}; - } - - /* (non-Javadoc) - * @see org.archive.util.Reporter#reportTo(java.io.Writer) - */ - public void reportTo(PrintWriter writer) { - reportTo(null,writer); - } - + @Override public synchronized Map shortReportMap() { Map map = new LinkedHashMap(); @@ -498,6 +485,7 @@ public abstract class WorkQueue implements Frontier.FrontierGroup, return sessionBudget - (totalExpenditure-expenditureAtLastActivation); } + @Override public synchronized void shortReportLineTo(PrintWriter writer) { // queue name writer.print(classKey); @@ -545,6 +533,7 @@ public abstract class WorkQueue implements Frontier.FrontierGroup, writer.print("\n"); } + @Override public String shortReportLegend() { return "queue precedence currentSize totalEnqueues sessionBalance " + "lastCost (averageCost) lastDequeueTime wakeTime " + @@ -559,7 +548,8 @@ public abstract class WorkQueue implements Frontier.FrontierGroup, * @param writer * @throws IOException */ - public synchronized void reportTo(String name, PrintWriter writer) { + @Override + public synchronized void reportTo(PrintWriter writer) { // name is ignored: only one kind of report for now writer.print("Queue "); writer.print(classKey); diff --git a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java index 6dc67082..f6ac6773 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java @@ -1086,15 +1086,8 @@ implements Closeable, // Reporter implementation // - public static String STANDARD_REPORT = "standard"; - public static String ALL_NONEMPTY = "nonempty"; - public static String ALL_QUEUES = "all"; - protected static String[] REPORTS = {STANDARD_REPORT,ALL_NONEMPTY,ALL_QUEUES}; - - public String[] getReports() { - return REPORTS; - } + @Override public Map shortReportMap() { if (this.allQueues == null) { return null; @@ -1128,6 +1121,7 @@ implements Closeable, /** * @param w Where to write to. */ + @Override public void shortReportLineTo(PrintWriter w) { if (!isRunning()) return; //??? @@ -1209,6 +1203,7 @@ implements Closeable, /* (non-Javadoc) * @see org.archive.util.Reporter#singleLineLegend() */ + @Override public String shortReportLegend() { return "total active in-process ready snoozed inactive retired exhausted"; } @@ -1219,27 +1214,132 @@ implements Closeable, * @param name Name of report. * @param writer Where to write to. */ - public synchronized void reportTo(String name, PrintWriter writer) { - if(ALL_NONEMPTY.equals(name)) { - allNonemptyReportTo(writer); - return; + @Override + public synchronized void reportTo(PrintWriter writer) { + int allCount = allQueues.size(); + int inProcessCount = inProcessQueues.size(); + int readyCount = readyClassQueues.size(); + int snoozedCount = getSnoozedCount(); + int activeCount = inProcessCount + readyCount + snoozedCount; + int inactiveCount = getTotalInactiveQueues(); + int retiredCount = getRetiredQueues().size(); + int exhaustedCount = + allCount - activeCount - inactiveCount - retiredCount; + + writer.print("Frontier report - "); + writer.print(ArchiveUtils.get12DigitDate()); + writer.print("\n"); + writer.print(" Job being crawled: "); + writer.print(controller.getMetadata().getJobName()); + writer.print("\n"); + writer.print("\n -----===== STATS =====-----\n"); + writer.print(" Discovered: "); + writer.print(Long.toString(discoveredUriCount())); + writer.print("\n"); + writer.print(" Queued: "); + writer.print(Long.toString(queuedUriCount())); + writer.print("\n"); + writer.print(" Finished: "); + writer.print(Long.toString(finishedUriCount())); + writer.print("\n"); + writer.print(" Successfully: "); + writer.print(Long.toString(succeededFetchCount())); + writer.print("\n"); + writer.print(" Failed: "); + writer.print(Long.toString(failedFetchCount())); + writer.print("\n"); + writer.print(" Disregarded: "); + writer.print(Long.toString(disregardedUriCount())); + writer.print("\n"); + writer.print("\n -----===== QUEUES =====-----\n"); + writer.print(" Already included size: "); + writer.print(Long.toString(uriUniqFilter.count())); + writer.print("\n"); + writer.print(" pending: "); + writer.print(Long.toString(uriUniqFilter.pending())); + writer.print("\n"); + writer.print("\n All class queues map size: "); + writer.print(Long.toString(allCount)); + writer.print("\n"); + writer.print( " Active queues: "); + writer.print(activeCount); + writer.print("\n"); + writer.print(" In-process: "); + writer.print(inProcessCount); + writer.print("\n"); + writer.print(" Ready: "); + writer.print(readyCount); + writer.print("\n"); + writer.print(" Snoozed: "); + writer.print(snoozedCount); + writer.print("\n"); + writer.print(" Inactive queues: "); + writer.print(inactiveCount); + writer.print(" ("); + Map> inactives = getInactiveQueuesByPrecedence(); + boolean betwixt = false; + for(Integer k : inactives.keySet()) { + if(betwixt) { + writer.print("; "); + } + writer.print("p"); + writer.print(k); + writer.print(": "); + writer.print(inactives.get(k).size()); + betwixt = true; } - if(ALL_QUEUES.equals(name)) { - allQueuesReportTo(writer); - return; + writer.print(")\n"); + writer.print(" Retired queues: "); + writer.print(retiredCount); + writer.print("\n"); + writer.print(" Exhausted queues: "); + writer.print(exhaustedCount); + writer.print("\n"); + + State last = lastReachedState; + writer.print("\n Last state: "+last); + + writer.print("\n -----===== MANAGER THREAD =====-----\n"); + ToeThread.reportThread(managerThread, writer); + + writer.print("\n -----===== "+largestQueues.size()+" LONGEST QUEUES =====-----\n"); + appendQueueReports(writer, "LONGEST", largestQueues.getEntriesDescending().iterator(), largestQueues.size(), largestQueues.size()); + + writer.print("\n -----===== IN-PROCESS QUEUES =====-----\n"); + Collection inProcess = inProcessQueues; + ArrayList copy = extractSome(inProcess, maxQueuesPerReportCategory); + appendQueueReports(writer, "IN-PROCESS", copy.iterator(), copy.size(), maxQueuesPerReportCategory); + + writer.print("\n -----===== READY QUEUES =====-----\n"); + appendQueueReports(writer, "READY", this.readyClassQueues.iterator(), + this.readyClassQueues.size(), maxQueuesPerReportCategory); + + writer.print("\n -----===== SNOOZED QUEUES =====-----\n"); + Object[] objs = snoozedClassQueues.toArray(); + DelayedWorkQueue[] qs = Arrays.copyOf(objs,objs.length,DelayedWorkQueue[].class); + Arrays.sort(qs); + appendQueueReports(writer, "SNOOZED", new ObjectArrayIterator(qs), getSnoozedCount(), maxQueuesPerReportCategory); + + writer.print("\n -----===== INACTIVE QUEUES =====-----\n"); + SortedMap> sortedInactives = getInactiveQueuesByPrecedence(); + for(Integer prec : sortedInactives.keySet()) { + Queue inactiveQueues = sortedInactives.get(prec); + appendQueueReports(writer, "INACTIVE-p"+prec, inactiveQueues.iterator(), + inactiveQueues.size(), maxQueuesPerReportCategory); } - if(name!=null && !STANDARD_REPORT.equals(name)) { - writer.print(name); - writer.print(" unavailable; standard report:\n"); - } - standardReportTo(writer); - } + + writer.print("\n -----===== RETIRED QUEUES =====-----\n"); + appendQueueReports(writer, "RETIRED", getRetiredQueues().iterator(), + getRetiredQueues().size(), maxQueuesPerReportCategory); + + writer.flush(); + } /** Compact report of all nonempty queues (one queue per line) * * @param writer */ - private void allNonemptyReportTo(PrintWriter writer) { + public void allNonemptyReportTo(PrintWriter writer) { ArrayList inProcessQueuesCopy; synchronized(this.inProcessQueues) { // grab a copy that will be stable against mods for report duration @@ -1269,7 +1369,7 @@ implements Closeable, * * @param writer */ - private void allQueuesReportTo(PrintWriter writer) { + public void allQueuesReportTo(PrintWriter writer) { queueSingleLinesTo(writer, allQueues.keySet().iterator()); } @@ -1314,130 +1414,6 @@ implements Closeable, } } - /** - * @param w Writer to print to. - */ - private void standardReportTo(PrintWriter w) { - int allCount = allQueues.size(); - int inProcessCount = inProcessQueues.size(); - int readyCount = readyClassQueues.size(); - int snoozedCount = getSnoozedCount(); - int activeCount = inProcessCount + readyCount + snoozedCount; - int inactiveCount = getTotalInactiveQueues(); - int retiredCount = getRetiredQueues().size(); - int exhaustedCount = - allCount - activeCount - inactiveCount - retiredCount; - - w.print("Frontier report - "); - w.print(ArchiveUtils.get12DigitDate()); - w.print("\n"); - w.print(" Job being crawled: "); - w.print(controller.getMetadata().getJobName()); - w.print("\n"); - w.print("\n -----===== STATS =====-----\n"); - w.print(" Discovered: "); - w.print(Long.toString(discoveredUriCount())); - w.print("\n"); - w.print(" Queued: "); - w.print(Long.toString(queuedUriCount())); - w.print("\n"); - w.print(" Finished: "); - w.print(Long.toString(finishedUriCount())); - w.print("\n"); - w.print(" Successfully: "); - w.print(Long.toString(succeededFetchCount())); - w.print("\n"); - w.print(" Failed: "); - w.print(Long.toString(failedFetchCount())); - w.print("\n"); - w.print(" Disregarded: "); - w.print(Long.toString(disregardedUriCount())); - w.print("\n"); - w.print("\n -----===== QUEUES =====-----\n"); - w.print(" Already included size: "); - w.print(Long.toString(uriUniqFilter.count())); - w.print("\n"); - w.print(" pending: "); - w.print(Long.toString(uriUniqFilter.pending())); - w.print("\n"); - w.print("\n All class queues map size: "); - w.print(Long.toString(allCount)); - w.print("\n"); - w.print( " Active queues: "); - w.print(activeCount); - w.print("\n"); - w.print(" In-process: "); - w.print(inProcessCount); - w.print("\n"); - w.print(" Ready: "); - w.print(readyCount); - w.print("\n"); - w.print(" Snoozed: "); - w.print(snoozedCount); - w.print("\n"); - w.print(" Inactive queues: "); - w.print(inactiveCount); - w.print(" ("); - Map> inactives = getInactiveQueuesByPrecedence(); - boolean betwixt = false; - for(Integer k : inactives.keySet()) { - if(betwixt) { - w.print("; "); - } - w.print("p"); - w.print(k); - w.print(": "); - w.print(inactives.get(k).size()); - betwixt = true; - } - w.print(")\n"); - w.print(" Retired queues: "); - w.print(retiredCount); - w.print("\n"); - w.print(" Exhausted queues: "); - w.print(exhaustedCount); - w.print("\n"); - - State last = lastReachedState; - w.print("\n Last state: "+last); - - w.print("\n -----===== MANAGER THREAD =====-----\n"); - ToeThread.reportThread(managerThread, w); - - w.print("\n -----===== "+largestQueues.size()+" LONGEST QUEUES =====-----\n"); - appendQueueReports(w, "LONGEST", largestQueues.getEntriesDescending().iterator(), largestQueues.size(), largestQueues.size()); - - w.print("\n -----===== IN-PROCESS QUEUES =====-----\n"); - Collection inProcess = inProcessQueues; - ArrayList copy = extractSome(inProcess, maxQueuesPerReportCategory); - appendQueueReports(w, "IN-PROCESS", copy.iterator(), copy.size(), maxQueuesPerReportCategory); - - w.print("\n -----===== READY QUEUES =====-----\n"); - appendQueueReports(w, "READY", this.readyClassQueues.iterator(), - this.readyClassQueues.size(), maxQueuesPerReportCategory); - - w.print("\n -----===== SNOOZED QUEUES =====-----\n"); - Object[] objs = snoozedClassQueues.toArray(); - DelayedWorkQueue[] qs = Arrays.copyOf(objs,objs.length,DelayedWorkQueue[].class); - Arrays.sort(qs); - appendQueueReports(w, "SNOOZED", new ObjectArrayIterator(qs), getSnoozedCount(), maxQueuesPerReportCategory); - - w.print("\n -----===== INACTIVE QUEUES =====-----\n"); - SortedMap> sortedInactives = getInactiveQueuesByPrecedence(); - for(Integer prec : sortedInactives.keySet()) { - Queue inactiveQueues = sortedInactives.get(prec); - appendQueueReports(w, "INACTIVE-p"+prec, inactiveQueues.iterator(), - inactiveQueues.size(), maxQueuesPerReportCategory); - } - - w.print("\n -----===== RETIRED QUEUES =====-----\n"); - appendQueueReports(w, "RETIRED", getRetiredQueues().iterator(), - getRetiredQueues().size(), maxQueuesPerReportCategory); - - w.flush(); - } - - /** * Extract some of the elements in the given collection to an * ArrayList. This method synchronizes on the given collection's diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceProvider.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceProvider.java index 5ecf02ba..65ab891d 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceProvider.java +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceProvider.java @@ -18,22 +18,22 @@ */ package org.archive.crawler.frontier.precedence; -import java.io.PrintWriter; -import java.io.Serializable; +import java.io.PrintWriter; +import java.io.Serializable; import java.util.LinkedHashMap; import java.util.Map; - -import org.archive.modules.CrawlURI; -import org.archive.modules.fetcher.FetchStats; -import org.archive.modules.fetcher.FetchStats.Stage; -import org.archive.util.ArchiveUtils; -import org.archive.util.MultiReporter; + +import org.archive.modules.CrawlURI; +import org.archive.modules.fetcher.FetchStats; +import org.archive.modules.fetcher.FetchStats.Stage; +import org.archive.util.ArchiveUtils; +import org.archive.util.Reporter; /** * Parent class for precedence-providers, stateful helpers that can be * installed in a WorkQueue to implement various queue-precedence policies. */ -abstract public class PrecedenceProvider implements MultiReporter, +abstract public class PrecedenceProvider implements Reporter, FetchStats.CollectsFetchStats, Serializable { private static final long serialVersionUID = 1L; @@ -47,41 +47,32 @@ FetchStats.CollectsFetchStats, Serializable { // by default do nothing; subclasses do more } - public String[] getReports() { - // TODO Auto-generated method stub - return null; - } - - /* (non-Javadoc) - * @see org.archive.util.Reporter#reportTo(java.lang.String, java.io.PrintWriter) - */ - public void reportTo(String name, PrintWriter writer) { - // name ignored, only one report - writer.println(shortReportLegend()); - shortReportLineTo(writer); - } - /* (non-Javadoc) * @see org.archive.util.Reporter#reportTo(java.io.PrintWriter) - */ - public void reportTo(PrintWriter writer) { - reportTo(null,writer); + */ + @Override + public void reportTo(PrintWriter writer) { + writer.println(shortReportLegend()); + shortReportLineTo(writer); } - + + @Override public String shortReportLegend() { return getClass().getSimpleName(); } - + public String shortReportLine() { return ArchiveUtils.shortReportLine(this); } - + + @Override public Map shortReportMap() { Map data = new LinkedHashMap(); data.put("precedence", getPrecedence()); return data; } + @Override public void shortReportLineTo(PrintWriter writer) { writer.print(getPrecedence()); } diff --git a/engine/src/main/java/org/archive/crawler/reporting/FrontierNonemptyReport.java b/engine/src/main/java/org/archive/crawler/reporting/FrontierNonemptyReport.java index 1228045b..7c370f51 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/FrontierNonemptyReport.java +++ b/engine/src/main/java/org/archive/crawler/reporting/FrontierNonemptyReport.java @@ -19,6 +19,7 @@ package org.archive.crawler.reporting; +import java.io.IOException; import java.io.PrintWriter; import org.archive.crawler.frontier.WorkQueueFrontier; @@ -37,9 +38,14 @@ public class FrontierNonemptyReport extends Report { writer.println("frontier unstarted"); } else if (stats.controller.getFrontier().isEmpty()) { writer.println("frontier empty"); + } else if (stats.controller.getFrontier() instanceof WorkQueueFrontier) { + ((WorkQueueFrontier)stats.controller.getFrontier()).allNonemptyReportTo(writer); } else { - stats.controller.getFrontier().reportTo( - WorkQueueFrontier.ALL_NONEMPTY, writer); + try { + stats.controller.getFrontier().reportTo(writer); + } catch (IOException e) { + e.printStackTrace(); + } } } diff --git a/engine/src/main/java/org/archive/crawler/reporting/FrontierSummaryReport.java b/engine/src/main/java/org/archive/crawler/reporting/FrontierSummaryReport.java index 9b54dcd8..f46cf213 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/FrontierSummaryReport.java +++ b/engine/src/main/java/org/archive/crawler/reporting/FrontierSummaryReport.java @@ -19,10 +19,9 @@ package org.archive.crawler.reporting; +import java.io.IOException; import java.io.PrintWriter; -import org.archive.crawler.frontier.WorkQueueFrontier; - /** * Frontier summary report showing a limited number of queues of each * type -- as typically consulted during a crawl in progress. @@ -35,11 +34,12 @@ public class FrontierSummaryReport extends Report { public void write(PrintWriter writer, StatisticsTracker stats) { if(!stats.controller.getFrontier().isRunning()) { writer.println("frontier unstarted"); -// } else if (stats.controller.getFrontier().isEmpty()) { -// writer.println("frontier empty"); } else { - stats.controller.getFrontier().reportTo( - WorkQueueFrontier.STANDARD_REPORT, writer); + try { + stats.controller.getFrontier().reportTo(writer); + } catch (IOException e) { + e.printStackTrace(); + } } } diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index 26d4f5b0..cfc01bf4 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -91,8 +91,8 @@ import org.archive.spring.OverlayContext; import org.archive.spring.OverlayMapsSource; import org.archive.util.ArchiveUtils; import org.archive.util.Base32; -import org.archive.util.MultiReporter; import org.archive.util.Recorder; +import org.archive.util.Reporter; import org.json.JSONException; import org.json.JSONObject; @@ -110,7 +110,7 @@ import org.json.JSONObject; * @author Gordon Mohr */ public class CrawlURI -implements MultiReporter, Serializable, OverlayContext { +implements Reporter, Serializable, OverlayContext { private static final long serialVersionUID = 3L; private static final Logger logger = @@ -1418,6 +1418,7 @@ implements MultiReporter, Serializable, OverlayContext { return ArchiveUtils.shortReportLine(this); } + @Override public Map shortReportMap() { Map map = new LinkedHashMap(); map.put("class", getClass().getName()); @@ -1427,6 +1428,7 @@ implements MultiReporter, Serializable, OverlayContext { return map; } + @Override public void shortReportLineTo(PrintWriter w) { String className = this.getClass().getName(); className = className.substring(className.lastIndexOf(".")+1); @@ -1442,31 +1444,18 @@ implements MultiReporter, Serializable, OverlayContext { /* (non-Javadoc) * @see org.archive.util.Reporter#singleLineLegend() */ + @Override public String shortReportLegend() { return "className uri hopsPath viaUri"; } - - /* (non-Javadoc) - * @see org.archive.util.Reporter#getReports() - */ - public String[] getReports() { - // none but default: empty options - return new String[] {}; - } - - /* (non-Javadoc) - * @see org.archive.util.Reporter#reportTo(java.lang.String, java.io.Writer) - */ - public void reportTo(String name, PrintWriter writer) { - shortReportLineTo(writer); - writer.print("\n"); - } /* (non-Javadoc) * @see org.archive.util.Reporter#reportTo(java.io.Writer) */ + @Override public void reportTo(PrintWriter writer) throws IOException { - reportTo(null,writer); + shortReportLineTo(writer); + writer.print("\n"); } diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java b/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java index f1c98511..85f489a5 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java @@ -23,11 +23,11 @@ import java.io.Serializable; import java.util.LinkedHashMap; import java.util.Map; +import org.apache.commons.httpclient.HttpStatus; import org.archive.modules.CrawlURI; -import org.archive.util.ArchiveUtils; -import org.archive.util.MultiReporter; -import org.apache.commons.httpclient.HttpStatus; import org.archive.modules.deciderules.recrawl.IdenticalDigestDecideRule; +import org.archive.util.ArchiveUtils; +import org.archive.util.Reporter; /** * Collector of statistics for a 'subset' of a crawl, @@ -36,7 +36,7 @@ import org.archive.modules.deciderules.recrawl.IdenticalDigestDecideRule; * * @author gojomo */ -public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter { +public class FetchStats implements Serializable, FetchStatusCodes, Reporter { private static final long serialVersionUID = 8624425657056569036L; public enum Stage {SCHEDULED, RELOCATED, RETRIED, @@ -185,29 +185,18 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter public long getDupByHashUrls() { return dupByHashUrls; - } - - public String[] getReports() { - // TODO Auto-generated method stub - return null; - } - - /* (non-Javadoc) - * @see org.archive.util.Reporter#reportTo(java.lang.String, java.io.PrintWriter) - */ - public void reportTo(String name, PrintWriter writer) { - // name ignored, only one report - writer.println(shortReportLegend()); - shortReportLineTo(writer); } /* (non-Javadoc) * @see org.archive.util.Reporter#reportTo(java.io.PrintWriter) */ + @Override // Reporter public void reportTo(PrintWriter writer) { - reportTo(null,writer); + writer.println(shortReportLegend()); + shortReportLineTo(writer); } + @Override public String shortReportLegend() { return "totalScheduled fetchSuccesses fetchFailures fetchDisregards " + "fetchResponses robotsDenials successBytes totalBytes " + @@ -218,6 +207,7 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter return ArchiveUtils.shortReportLine(this); } + @Override public void shortReportLineTo(PrintWriter writer) { writer.print(totalScheduled); writer.print(" "); @@ -240,6 +230,7 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter writer.print(ArchiveUtils.getLog17Date(lastSuccessTime)); } + @Override public Map shortReportMap() { Map map = new LinkedHashMap(); map.put("totalScheduled", totalScheduled);