lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-07-18 21:47:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into no-package-scope

Browse Source
This commit is contained in:
Noah Levitt
2012-06-05 17:58:40 -07:00
parent 3dce1e909d ee1e25ca69
commit 35378df2fa
16 changed files with 208 additions and 367 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/src/main/java/org/archive/util/ArchiveUtils.java
+2 -18
View File
@@ -741,22 +741,6 @@ public class ArchiveUtils {
pw.flush();
return sw.toString();
}
/**
* Compose the requested report into a String. DANGEROUS IF REPORT
* CAN BE LARGE.
*
* @param rep Reported
* @param name String name of report to compose
* @return String of report
*/
public static String writeReportToString(MultiReporter rep, String name) {
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
rep.reportTo(name,pw);
pw.flush();
return sw.toString();
}
/**
* Enhance given object's default String display for appearing
@@ -773,8 +757,8 @@ public class ArchiveUtils {
} else if (obj instanceof Map) {
return prettyString((Map) obj);
} else {
return "<"+obj+">";
}
return "<"+obj+">";
}
}
/**
commons/src/main/java/org/archive/util/DevUtils.java
+2 -2
View File
@@ -55,8 +55,8 @@ public class DevUtils {
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
final Thread current = Thread.currentThread();
if (current instanceof MultiReporter) {
MultiReporter tt = (MultiReporter)current;
if (current instanceof Reporter) {
Reporter tt = (Reporter)current;
try {
tt.reportTo(pw);
} catch (IOException e) {
commons/src/main/java/org/archive/util/MultiReporter.java
-46
View File
@@ -1,46 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.util;
import java.io.PrintWriter;
/**
* @contributor stack
*/
public interface MultiReporter extends Reporter {
/**
* Get an array of report names offered by this Reporter.
* A name in brackets indicates a free-form String,
* in accordance with the informal description inside
* the brackets, may yield a useful report.
*
* @return String array of report names, empty if there is only
* one report type
*/
public String[] getReports();
/**
* Make a report of the given name to the passed-in Writer,
* If null, give the default report.
*
* @param writer to receive report
*/
public void reportTo(String name, PrintWriter writer);
}
commons/src/main/java/org/archive/util/Reporter.java
+1
View File
@@ -37,6 +37,7 @@ public interface Reporter {
*
* @param writer to receive report
*/
@Deprecated
public void shortReportLineTo(PrintWriter pw) throws IOException;
engine/src/main/java/org/archive/crawler/framework/Frontier.java
+2 -2
View File
@@ -28,7 +28,7 @@ import org.archive.modules.CrawlURI;
import org.archive.modules.deciderules.DecideRule;
import org.archive.modules.fetcher.FetchStats;
import org.archive.util.IdentityCacheable;
import org.archive.util.MultiReporter;
import org.archive.util.Reporter;
import org.json.JSONException;
import org.springframework.context.Lifecycle;
@@ -101,7 +101,7 @@ import org.springframework.context.Lifecycle;
* @see org.archive.crawler.framework.ToeThread
* @see org.archive.crawler.settings.ModuleType
*/
public interface Frontier extends Lifecycle, MultiReporter {
public interface Frontier extends Lifecycle, Reporter {
/**
* Get the next URI that should be processed. If no URI becomes availible
engine/src/main/java/org/archive/crawler/framework/ToePool.java
+11 -38
View File
@@ -23,14 +23,14 @@ import java.io.PrintWriter;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.TreeSet;
import java.util.Map.Entry;
import java.util.TreeSet;
import org.apache.commons.lang.StringUtils;
import org.archive.crawler.reporting.AlertThreadGroup;
import org.archive.util.ArchiveUtils;
import org.archive.util.Histotable;
import org.archive.util.MultiReporter;
import org.archive.util.Reporter;
/**
* A collection of ToeThreads. The class manages the ToeThreads currently
@@ -43,7 +43,7 @@ import org.archive.util.MultiReporter;
*
* @see org.archive.crawler.framework.ToeThread
*/
public class ToePool extends ThreadGroup implements MultiReporter {
public class ToePool extends ThreadGroup implements Reporter {
/** run worker thread slightly lower than usual */
public static int DEFAULT_TOE_PRIORITY = Thread.NORM_PRIORITY - 1;
@@ -70,7 +70,6 @@ public class ToePool extends ThreadGroup implements MultiReporter {
toe.interrupt();
}
}
// this.controller = null;
}
/**
@@ -190,37 +189,15 @@ public class ToePool extends ThreadGroup implements MultiReporter {
// Reporter implementation
//
public static String STANDARD_REPORT = "standard";
public static String COMPACT_REPORT = "compact";
protected static String[] REPORTS = {STANDARD_REPORT,COMPACT_REPORT};
public String[] getReports() {
return REPORTS;
}
public void reportTo(String name, PrintWriter writer) {
if(COMPACT_REPORT.equals(name)) {
compactReportTo(writer);
return;
}
if(name!=null && !STANDARD_REPORT.equals(name)) {
writer.print(name);
writer.print(" not recognized: giving standard report/n");
}
standardReportTo(writer);
}
/* (non-Javadoc)
* @see org.archive.util.Reporter#reportTo(java.io.Writer)
*/
protected void standardReportTo(PrintWriter writer) {
@Override
public void reportTo(PrintWriter writer) {
writer.print("Toe threads report - "
+ ArchiveUtils.get12DigitDate() + "\n");
writer.print(" Job being crawled: "
+ this.controller.getMetadata().getJobName() + "\n");
writer.print(" Number of toe threads in pool: " + getToeCount() + " ("
+ getActiveToeCount() + " active)\n\n");
Thread[] toes = this.getToes();
synchronized (toes) {
for (int i = 0; i < toes.length; i++) {
@@ -233,12 +210,9 @@ public class ToePool extends ThreadGroup implements MultiReporter {
}
}
}
}
}
/* (non-Javadoc)
* @see org.archive.util.Reporter#reportTo(java.io.Writer)
*/
protected void compactReportTo(PrintWriter writer) {
public void compactReportTo(PrintWriter writer) {
writer.print(getToeCount() + " threads (" + getActiveToeCount()
+ " active)\n");
@@ -262,6 +236,7 @@ public class ToePool extends ThreadGroup implements MultiReporter {
}
}
@Override
public Map<String, Object> shortReportMap() {
Histotable<Object> steps = new Histotable<Object>();
Histotable<Object> processors = new Histotable<Object>();
@@ -291,6 +266,7 @@ public class ToePool extends ThreadGroup implements MultiReporter {
}
@SuppressWarnings("unchecked")
@Override
public void shortReportLineTo(PrintWriter w) {
Map<String, Object> map = shortReportMap();
w.print(map.get("toeCount"));
@@ -327,14 +303,11 @@ public class ToePool extends ThreadGroup implements MultiReporter {
/* (non-Javadoc)
* @see org.archive.util.Reporter#singleLineLegend()
*/
@Override
public String shortReportLegend() {
return "total: mostCommonStateTotal secondMostCommonStateTotal";
}
public void reportTo(PrintWriter writer) {
reportTo(null,writer);
}
public void waitForAll() {
while (true) try {
engine/src/main/java/org/archive/crawler/framework/ToeThread.java
+7 -16
View File
@@ -42,9 +42,9 @@ import org.archive.modules.fetcher.HostResolver;
import org.archive.spring.KeyedProperties;
import org.archive.util.ArchiveUtils;
import org.archive.util.DevUtils;
import org.archive.util.MultiReporter;
import org.archive.util.ProgressStatisticsReporter;
import org.archive.util.Recorder;
import org.archive.util.Reporter;
import com.sleepycat.util.RuntimeExceptionWrapper;
@@ -55,7 +55,7 @@ import com.sleepycat.util.RuntimeExceptionWrapper;
* @author Gordon Mohr
*/
public class ToeThread extends Thread
implements MultiReporter, ProgressStatisticsReporter,
implements Reporter, ProgressStatisticsReporter,
HostResolver, SinkHandlerLogThread, ChainStatusReceiver {
public enum Step {
@@ -396,10 +396,10 @@ implements MultiReporter, ProgressStatisticsReporter,
/**
* Compiles and returns a report on its status.
* @param name Report name.
* @param pw Where to print.
*/
public void reportTo(String name, PrintWriter pw) {
@Override
public void reportTo(PrintWriter pw) {
// name is ignored for now: only one kind of report
pw.print("[");
@@ -488,6 +488,7 @@ implements MultiReporter, ProgressStatisticsReporter,
}
}
@Override
public Map<String, Object> shortReportMap() {
Map<String,Object> data = new LinkedHashMap<String, Object>();
data.put("serialNumber", serialNumber);
@@ -518,6 +519,7 @@ implements MultiReporter, ProgressStatisticsReporter,
/**
* @param w PrintWriter to write to.
*/
@Override
public void shortReportLineTo(PrintWriter w)
{
w.print("#");
@@ -565,21 +567,10 @@ implements MultiReporter, ProgressStatisticsReporter,
w.flush();
}
@Override
public String shortReportLegend() {
return "#serialNumber processorName currentUri (fetchAttempts) threadState threadStep";
}
/* (non-Javadoc)
* @see org.archive.util.Reporter#getReports()
*/
public String[] getReports() {
// for now none but the default
return new String[] {};
}
public void reportTo(PrintWriter writer) {
reportTo(null, writer);
}
public String shortReportLine() {
return ArchiveUtils.shortReportLine(this);
engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java
-5
View File
@@ -39,7 +39,6 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.util.Collection;
import java.util.Iterator;
import java.util.concurrent.atomic.AtomicLong;
@@ -1131,10 +1130,6 @@ public abstract class AbstractFrontier
return ArchiveUtils.shortReportLine(this);
}
public void reportTo(PrintWriter writer) {
reportTo(null, writer);
}
public void onApplicationEvent(ApplicationEvent event) {
if(event instanceof CrawlStateEvent) {
CrawlStateEvent event1 = (CrawlStateEvent)event;
engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java
+1 -1
View File
@@ -21,9 +21,9 @@ package org.archive.crawler.frontier;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.Map.Entry;
import java.util.Queue;
import java.util.SortedMap;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.DelayQueue;
import java.util.concurrent.LinkedBlockingQueue;
engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java
+7 -17
View File
@@ -38,8 +38,8 @@ import org.archive.modules.fetcher.FetchStats;
import org.archive.modules.fetcher.FetchStats.Stage;
import org.archive.util.ArchiveUtils;
import org.archive.util.IdentityCacheable;
import org.archive.util.MultiReporter;
import org.archive.util.ObjectIdentityCache;
import org.archive.util.Reporter;
/**
* A single queue of related URIs to visit, grouped by a classKey
@@ -49,7 +49,7 @@ import org.archive.util.ObjectIdentityCache;
* @author Christian Kohlschuetter
*/
public abstract class WorkQueue implements Frontier.FrontierGroup,
Serializable, MultiReporter, Delayed, IdentityCacheable {
Serializable, Reporter, Delayed, IdentityCacheable {
private static final long serialVersionUID = -3199666138837266341L;
private static final Logger logger =
Logger.getLogger(WorkQueue.class.getName());
@@ -451,20 +451,7 @@ public abstract class WorkQueue implements Frontier.FrontierGroup,
// Reporter
//
/* (non-Javadoc)
* @see org.archive.util.Reporter#getReports()
*/
public String[] getReports() {
return new String[] {};
}
/* (non-Javadoc)
* @see org.archive.util.Reporter#reportTo(java.io.Writer)
*/
public void reportTo(PrintWriter writer) {
reportTo(null,writer);
}
@Override
public synchronized Map<String, Object> shortReportMap() {
Map<String,Object> map = new LinkedHashMap<String, Object>();
@@ -498,6 +485,7 @@ public abstract class WorkQueue implements Frontier.FrontierGroup,
return sessionBudget - (totalExpenditure-expenditureAtLastActivation);
}
@Override
public synchronized void shortReportLineTo(PrintWriter writer) {
// queue name
writer.print(classKey);
@@ -545,6 +533,7 @@ public abstract class WorkQueue implements Frontier.FrontierGroup,
writer.print("\n");
}
@Override
public String shortReportLegend() {
return "queue precedence currentSize totalEnqueues sessionBalance " +
"lastCost (averageCost) lastDequeueTime wakeTime " +
@@ -559,7 +548,8 @@ public abstract class WorkQueue implements Frontier.FrontierGroup,
* @param writer
* @throws IOException
*/
public synchronized void reportTo(String name, PrintWriter writer) {
@Override
public synchronized void reportTo(PrintWriter writer) {
// name is ignored: only one kind of report for now
writer.print("Queue ");
writer.print(classKey);
engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java
+123 -147
View File
@@ -1086,15 +1086,8 @@ implements Closeable,
// Reporter implementation
//
public static String STANDARD_REPORT = "standard";
public static String ALL_NONEMPTY = "nonempty";
public static String ALL_QUEUES = "all";
protected static String[] REPORTS = {STANDARD_REPORT,ALL_NONEMPTY,ALL_QUEUES};
public String[] getReports() {
return REPORTS;
}
@Override
public Map<String, Object> shortReportMap() {
if (this.allQueues == null) {
return null;
@@ -1128,6 +1121,7 @@ implements Closeable,
/**
* @param w Where to write to.
*/
@Override
public void shortReportLineTo(PrintWriter w) {
if (!isRunning()) return; //???
@@ -1209,6 +1203,7 @@ implements Closeable,
/* (non-Javadoc)
* @see org.archive.util.Reporter#singleLineLegend()
*/
@Override
public String shortReportLegend() {
return "total active in-process ready snoozed inactive retired exhausted";
}
@@ -1219,27 +1214,132 @@ implements Closeable,
* @param name Name of report.
* @param writer Where to write to.
*/
public synchronized void reportTo(String name, PrintWriter writer) {
if(ALL_NONEMPTY.equals(name)) {
allNonemptyReportTo(writer);
return;
@Override
public synchronized void reportTo(PrintWriter writer) {
int allCount = allQueues.size();
int inProcessCount = inProcessQueues.size();
int readyCount = readyClassQueues.size();
int snoozedCount = getSnoozedCount();
int activeCount = inProcessCount + readyCount + snoozedCount;
int inactiveCount = getTotalInactiveQueues();
int retiredCount = getRetiredQueues().size();
int exhaustedCount =
allCount - activeCount - inactiveCount - retiredCount;
writer.print("Frontier report - ");
writer.print(ArchiveUtils.get12DigitDate());
writer.print("\n");
writer.print(" Job being crawled: ");
writer.print(controller.getMetadata().getJobName());
writer.print("\n");
writer.print("\n -----===== STATS =====-----\n");
writer.print(" Discovered: ");
writer.print(Long.toString(discoveredUriCount()));
writer.print("\n");
writer.print(" Queued: ");
writer.print(Long.toString(queuedUriCount()));
writer.print("\n");
writer.print(" Finished: ");
writer.print(Long.toString(finishedUriCount()));
writer.print("\n");
writer.print(" Successfully: ");
writer.print(Long.toString(succeededFetchCount()));
writer.print("\n");
writer.print(" Failed: ");
writer.print(Long.toString(failedFetchCount()));
writer.print("\n");
writer.print(" Disregarded: ");
writer.print(Long.toString(disregardedUriCount()));
writer.print("\n");
writer.print("\n -----===== QUEUES =====-----\n");
writer.print(" Already included size: ");
writer.print(Long.toString(uriUniqFilter.count()));
writer.print("\n");
writer.print(" pending: ");
writer.print(Long.toString(uriUniqFilter.pending()));
writer.print("\n");
writer.print("\n All class queues map size: ");
writer.print(Long.toString(allCount));
writer.print("\n");
writer.print( " Active queues: ");
writer.print(activeCount);
writer.print("\n");
writer.print(" In-process: ");
writer.print(inProcessCount);
writer.print("\n");
writer.print(" Ready: ");
writer.print(readyCount);
writer.print("\n");
writer.print(" Snoozed: ");
writer.print(snoozedCount);
writer.print("\n");
writer.print(" Inactive queues: ");
writer.print(inactiveCount);
writer.print(" (");
Map<Integer,Queue<String>> inactives = getInactiveQueuesByPrecedence();
boolean betwixt = false;
for(Integer k : inactives.keySet()) {
if(betwixt) {
writer.print("; ");
}
writer.print("p");
writer.print(k);
writer.print(": ");
writer.print(inactives.get(k).size());
betwixt = true;
}
if(ALL_QUEUES.equals(name)) {
allQueuesReportTo(writer);
return;
writer.print(")\n");
writer.print(" Retired queues: ");
writer.print(retiredCount);
writer.print("\n");
writer.print(" Exhausted queues: ");
writer.print(exhaustedCount);
writer.print("\n");
State last = lastReachedState;
writer.print("\n Last state: "+last);
writer.print("\n -----===== MANAGER THREAD =====-----\n");
ToeThread.reportThread(managerThread, writer);
writer.print("\n -----===== "+largestQueues.size()+" LONGEST QUEUES =====-----\n");
appendQueueReports(writer, "LONGEST", largestQueues.getEntriesDescending().iterator(), largestQueues.size(), largestQueues.size());
writer.print("\n -----===== IN-PROCESS QUEUES =====-----\n");
Collection<WorkQueue> inProcess = inProcessQueues;
ArrayList<WorkQueue> copy = extractSome(inProcess, maxQueuesPerReportCategory);
appendQueueReports(writer, "IN-PROCESS", copy.iterator(), copy.size(), maxQueuesPerReportCategory);
writer.print("\n -----===== READY QUEUES =====-----\n");
appendQueueReports(writer, "READY", this.readyClassQueues.iterator(),
this.readyClassQueues.size(), maxQueuesPerReportCategory);
writer.print("\n -----===== SNOOZED QUEUES =====-----\n");
Object[] objs = snoozedClassQueues.toArray();
DelayedWorkQueue[] qs = Arrays.copyOf(objs,objs.length,DelayedWorkQueue[].class);
Arrays.sort(qs);
appendQueueReports(writer, "SNOOZED", new ObjectArrayIterator(qs), getSnoozedCount(), maxQueuesPerReportCategory);
writer.print("\n -----===== INACTIVE QUEUES =====-----\n");
SortedMap<Integer,Queue<String>> sortedInactives = getInactiveQueuesByPrecedence();
for(Integer prec : sortedInactives.keySet()) {
Queue<String> inactiveQueues = sortedInactives.get(prec);
appendQueueReports(writer, "INACTIVE-p"+prec, inactiveQueues.iterator(),
inactiveQueues.size(), maxQueuesPerReportCategory);
}
if(name!=null && !STANDARD_REPORT.equals(name)) {
writer.print(name);
writer.print(" unavailable; standard report:\n");
}
standardReportTo(writer);
}
writer.print("\n -----===== RETIRED QUEUES =====-----\n");
appendQueueReports(writer, "RETIRED", getRetiredQueues().iterator(),
getRetiredQueues().size(), maxQueuesPerReportCategory);
writer.flush();
}
/** Compact report of all nonempty queues (one queue per line)
*
* @param writer
*/
private void allNonemptyReportTo(PrintWriter writer) {
public void allNonemptyReportTo(PrintWriter writer) {
ArrayList<WorkQueue> inProcessQueuesCopy;
synchronized(this.inProcessQueues) {
// grab a copy that will be stable against mods for report duration
@@ -1269,7 +1369,7 @@ implements Closeable,
*
* @param writer
*/
private void allQueuesReportTo(PrintWriter writer) {
public void allQueuesReportTo(PrintWriter writer) {
queueSingleLinesTo(writer, allQueues.keySet().iterator());
}
@@ -1314,130 +1414,6 @@ implements Closeable,
}
}
/**
* @param w Writer to print to.
*/
private void standardReportTo(PrintWriter w) {
int allCount = allQueues.size();
int inProcessCount = inProcessQueues.size();
int readyCount = readyClassQueues.size();
int snoozedCount = getSnoozedCount();
int activeCount = inProcessCount + readyCount + snoozedCount;
int inactiveCount = getTotalInactiveQueues();
int retiredCount = getRetiredQueues().size();
int exhaustedCount =
allCount - activeCount - inactiveCount - retiredCount;
w.print("Frontier report - ");
w.print(ArchiveUtils.get12DigitDate());
w.print("\n");
w.print(" Job being crawled: ");
w.print(controller.getMetadata().getJobName());
w.print("\n");
w.print("\n -----===== STATS =====-----\n");
w.print(" Discovered: ");
w.print(Long.toString(discoveredUriCount()));
w.print("\n");
w.print(" Queued: ");
w.print(Long.toString(queuedUriCount()));
w.print("\n");
w.print(" Finished: ");
w.print(Long.toString(finishedUriCount()));
w.print("\n");
w.print(" Successfully: ");
w.print(Long.toString(succeededFetchCount()));
w.print("\n");
w.print(" Failed: ");
w.print(Long.toString(failedFetchCount()));
w.print("\n");
w.print(" Disregarded: ");
w.print(Long.toString(disregardedUriCount()));
w.print("\n");
w.print("\n -----===== QUEUES =====-----\n");
w.print(" Already included size: ");
w.print(Long.toString(uriUniqFilter.count()));
w.print("\n");
w.print(" pending: ");
w.print(Long.toString(uriUniqFilter.pending()));
w.print("\n");
w.print("\n All class queues map size: ");
w.print(Long.toString(allCount));
w.print("\n");
w.print( " Active queues: ");
w.print(activeCount);
w.print("\n");
w.print(" In-process: ");
w.print(inProcessCount);
w.print("\n");
w.print(" Ready: ");
w.print(readyCount);
w.print("\n");
w.print(" Snoozed: ");
w.print(snoozedCount);
w.print("\n");
w.print(" Inactive queues: ");
w.print(inactiveCount);
w.print(" (");
Map<Integer,Queue<String>> inactives = getInactiveQueuesByPrecedence();
boolean betwixt = false;
for(Integer k : inactives.keySet()) {
if(betwixt) {
w.print("; ");
}
w.print("p");
w.print(k);
w.print(": ");
w.print(inactives.get(k).size());
betwixt = true;
}
w.print(")\n");
w.print(" Retired queues: ");
w.print(retiredCount);
w.print("\n");
w.print(" Exhausted queues: ");
w.print(exhaustedCount);
w.print("\n");
State last = lastReachedState;
w.print("\n Last state: "+last);
w.print("\n -----===== MANAGER THREAD =====-----\n");
ToeThread.reportThread(managerThread, w);
w.print("\n -----===== "+largestQueues.size()+" LONGEST QUEUES =====-----\n");
appendQueueReports(w, "LONGEST", largestQueues.getEntriesDescending().iterator(), largestQueues.size(), largestQueues.size());
w.print("\n -----===== IN-PROCESS QUEUES =====-----\n");
Collection<WorkQueue> inProcess = inProcessQueues;
ArrayList<WorkQueue> copy = extractSome(inProcess, maxQueuesPerReportCategory);
appendQueueReports(w, "IN-PROCESS", copy.iterator(), copy.size(), maxQueuesPerReportCategory);
w.print("\n -----===== READY QUEUES =====-----\n");
appendQueueReports(w, "READY", this.readyClassQueues.iterator(),
this.readyClassQueues.size(), maxQueuesPerReportCategory);
w.print("\n -----===== SNOOZED QUEUES =====-----\n");
Object[] objs = snoozedClassQueues.toArray();
DelayedWorkQueue[] qs = Arrays.copyOf(objs,objs.length,DelayedWorkQueue[].class);
Arrays.sort(qs);
appendQueueReports(w, "SNOOZED", new ObjectArrayIterator(qs), getSnoozedCount(), maxQueuesPerReportCategory);
w.print("\n -----===== INACTIVE QUEUES =====-----\n");
SortedMap<Integer,Queue<String>> sortedInactives = getInactiveQueuesByPrecedence();
for(Integer prec : sortedInactives.keySet()) {
Queue<String> inactiveQueues = sortedInactives.get(prec);
appendQueueReports(w, "INACTIVE-p"+prec, inactiveQueues.iterator(),
inactiveQueues.size(), maxQueuesPerReportCategory);
}
w.print("\n -----===== RETIRED QUEUES =====-----\n");
appendQueueReports(w, "RETIRED", getRetiredQueues().iterator(),
getRetiredQueues().size(), maxQueuesPerReportCategory);
w.flush();
}
/**
* Extract some of the elements in the given collection to an
* ArrayList. This method synchronizes on the given collection's
engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceProvider.java
+20 -29
View File
@@ -18,22 +18,22 @@
*/
package org.archive.crawler.frontier.precedence;
import java.io.PrintWriter;
import java.io.Serializable;
import java.io.PrintWriter;
import java.io.Serializable;
import java.util.LinkedHashMap;
import java.util.Map;
import org.archive.modules.CrawlURI;
import org.archive.modules.fetcher.FetchStats;
import org.archive.modules.fetcher.FetchStats.Stage;
import org.archive.util.ArchiveUtils;
import org.archive.util.MultiReporter;
import org.archive.modules.CrawlURI;
import org.archive.modules.fetcher.FetchStats;
import org.archive.modules.fetcher.FetchStats.Stage;
import org.archive.util.ArchiveUtils;
import org.archive.util.Reporter;
/**
* Parent class for precedence-providers, stateful helpers that can be
* installed in a WorkQueue to implement various queue-precedence policies.
*/
abstract public class PrecedenceProvider implements MultiReporter,
abstract public class PrecedenceProvider implements Reporter,
FetchStats.CollectsFetchStats, Serializable {
private static final long serialVersionUID = 1L;
@@ -47,41 +47,32 @@ FetchStats.CollectsFetchStats, Serializable {
// by default do nothing; subclasses do more
}
public String[] getReports() {
// TODO Auto-generated method stub
return null;
}
/* (non-Javadoc)
* @see org.archive.util.Reporter#reportTo(java.lang.String, java.io.PrintWriter)
*/
public void reportTo(String name, PrintWriter writer) {
// name ignored, only one report
writer.println(shortReportLegend());
shortReportLineTo(writer);
}
/* (non-Javadoc)
* @see org.archive.util.Reporter#reportTo(java.io.PrintWriter)
*/
public void reportTo(PrintWriter writer) {
reportTo(null,writer);
*/
@Override
public void reportTo(PrintWriter writer) {
writer.println(shortReportLegend());
shortReportLineTo(writer);
}
@Override
public String shortReportLegend() {
return getClass().getSimpleName();
}
public String shortReportLine() {
return ArchiveUtils.shortReportLine(this);
}
@Override
public Map<String, Object> shortReportMap() {
Map<String,Object> data = new LinkedHashMap<String, Object>();
data.put("precedence", getPrecedence());
return data;
}
@Override
public void shortReportLineTo(PrintWriter writer) {
writer.print(getPrecedence());
}
engine/src/main/java/org/archive/crawler/reporting/FrontierNonemptyReport.java
+8 -2
View File
@@ -19,6 +19,7 @@
package org.archive.crawler.reporting;
import java.io.IOException;
import java.io.PrintWriter;
import org.archive.crawler.frontier.WorkQueueFrontier;
@@ -37,9 +38,14 @@ public class FrontierNonemptyReport extends Report {
writer.println("frontier unstarted");
} else if (stats.controller.getFrontier().isEmpty()) {
writer.println("frontier empty");
} else if (stats.controller.getFrontier() instanceof WorkQueueFrontier) {
((WorkQueueFrontier)stats.controller.getFrontier()).allNonemptyReportTo(writer);
} else {
stats.controller.getFrontier().reportTo(
WorkQueueFrontier.ALL_NONEMPTY, writer);
try {
stats.controller.getFrontier().reportTo(writer);
} catch (IOException e) {
e.printStackTrace();
}
}
}
engine/src/main/java/org/archive/crawler/reporting/FrontierSummaryReport.java
+6 -6
View File
@@ -19,10 +19,9 @@
package org.archive.crawler.reporting;
import java.io.IOException;
import java.io.PrintWriter;
import org.archive.crawler.frontier.WorkQueueFrontier;
/**
* Frontier summary report showing a limited number of queues of each
* type -- as typically consulted during a crawl in progress.
@@ -35,11 +34,12 @@ public class FrontierSummaryReport extends Report {
public void write(PrintWriter writer, StatisticsTracker stats) {
if(!stats.controller.getFrontier().isRunning()) {
writer.println("frontier unstarted");
// } else if (stats.controller.getFrontier().isEmpty()) {
// writer.println("frontier empty");
} else {
stats.controller.getFrontier().reportTo(
WorkQueueFrontier.STANDARD_REPORT, writer);
try {
stats.controller.getFrontier().reportTo(writer);
} catch (IOException e) {
e.printStackTrace();
}
}
}
modules/src/main/java/org/archive/modules/CrawlURI.java
+8 -19
View File
@@ -91,8 +91,8 @@ import org.archive.spring.OverlayContext;
import org.archive.spring.OverlayMapsSource;
import org.archive.util.ArchiveUtils;
import org.archive.util.Base32;
import org.archive.util.MultiReporter;
import org.archive.util.Recorder;
import org.archive.util.Reporter;
import org.json.JSONException;
import org.json.JSONObject;
@@ -110,7 +110,7 @@ import org.json.JSONObject;
* @author Gordon Mohr
*/
public class CrawlURI
implements MultiReporter, Serializable, OverlayContext {
implements Reporter, Serializable, OverlayContext {
private static final long serialVersionUID = 3L;
private static final Logger logger =
@@ -1418,6 +1418,7 @@ implements MultiReporter, Serializable, OverlayContext {
return ArchiveUtils.shortReportLine(this);
}
@Override
public Map<String, Object> shortReportMap() {
Map<String,Object> map = new LinkedHashMap<String, Object>();
map.put("class", getClass().getName());
@@ -1427,6 +1428,7 @@ implements MultiReporter, Serializable, OverlayContext {
return map;
}
@Override
public void shortReportLineTo(PrintWriter w) {
String className = this.getClass().getName();
className = className.substring(className.lastIndexOf(".")+1);
@@ -1442,31 +1444,18 @@ implements MultiReporter, Serializable, OverlayContext {
/* (non-Javadoc)
* @see org.archive.util.Reporter#singleLineLegend()
*/
@Override
public String shortReportLegend() {
return "className uri hopsPath viaUri";
}
/* (non-Javadoc)
* @see org.archive.util.Reporter#getReports()
*/
public String[] getReports() {
// none but default: empty options
return new String[] {};
}
/* (non-Javadoc)
* @see org.archive.util.Reporter#reportTo(java.lang.String, java.io.Writer)
*/
public void reportTo(String name, PrintWriter writer) {
shortReportLineTo(writer);
writer.print("\n");
}
/* (non-Javadoc)
* @see org.archive.util.Reporter#reportTo(java.io.Writer)
*/
@Override
public void reportTo(PrintWriter writer) throws IOException {
reportTo(null,writer);
shortReportLineTo(writer);
writer.print("\n");
}
modules/src/main/java/org/archive/modules/fetcher/FetchStats.java
+10 -19
View File
@@ -23,11 +23,11 @@ import java.io.Serializable;
import java.util.LinkedHashMap;
import java.util.Map;
import org.apache.commons.httpclient.HttpStatus;
import org.archive.modules.CrawlURI;
import org.archive.util.ArchiveUtils;
import org.archive.util.MultiReporter;
import org.apache.commons.httpclient.HttpStatus;
import org.archive.modules.deciderules.recrawl.IdenticalDigestDecideRule;
import org.archive.util.ArchiveUtils;
import org.archive.util.Reporter;
/**
* Collector of statistics for a 'subset' of a crawl,
@@ -36,7 +36,7 @@ import org.archive.modules.deciderules.recrawl.IdenticalDigestDecideRule;
*
* @author gojomo
*/
public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter {
public class FetchStats implements Serializable, FetchStatusCodes, Reporter {
private static final long serialVersionUID = 8624425657056569036L;
public enum Stage {SCHEDULED, RELOCATED, RETRIED,
@@ -185,29 +185,18 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter
public long getDupByHashUrls() {
return dupByHashUrls;
}
public String[] getReports() {
// TODO Auto-generated method stub
return null;
}
/* (non-Javadoc)
* @see org.archive.util.Reporter#reportTo(java.lang.String, java.io.PrintWriter)
*/
public void reportTo(String name, PrintWriter writer) {
// name ignored, only one report
writer.println(shortReportLegend());
shortReportLineTo(writer);
}
/* (non-Javadoc)
* @see org.archive.util.Reporter#reportTo(java.io.PrintWriter)
*/
@Override // Reporter
public void reportTo(PrintWriter writer) {
reportTo(null,writer);
writer.println(shortReportLegend());
shortReportLineTo(writer);
}
@Override
public String shortReportLegend() {
return "totalScheduled fetchSuccesses fetchFailures fetchDisregards " +
"fetchResponses robotsDenials successBytes totalBytes " +
@@ -218,6 +207,7 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter
return ArchiveUtils.shortReportLine(this);
}
@Override
public void shortReportLineTo(PrintWriter writer) {
writer.print(totalScheduled);
writer.print(" ");
@@ -240,6 +230,7 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter
writer.print(ArchiveUtils.getLog17Date(lastSuccessTime));
}
@Override
public Map<String, Object> shortReportMap() {
Map<String,Object> map = new LinkedHashMap<String, Object>();
map.put("totalScheduled", totalScheduled);