From bf638939b0ffabd124053dec04ce016cff131aec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristinn=20Sigur=C3=B0sson?= Date: Mon, 13 Jul 2015 14:32:34 +0000 Subject: [PATCH 1/5] Remove unused "top" hosts tracking. This used to be used for an overview report that was available in the H1 UI. Had become vestigial code that served no purpose. --- .../crawler/reporting/StatisticsTracker.java | 49 +------------------ 1 file changed, 1 insertion(+), 48 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java index 09af0b47..a8bb2e8e 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java +++ b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java @@ -50,7 +50,6 @@ import org.archive.crawler.event.StatSnapshotEvent; import org.archive.crawler.framework.CrawlController; import org.archive.crawler.framework.Engine; import org.archive.crawler.util.CrawledBytesHistotable; -import org.archive.crawler.util.TopNSet; import org.archive.modules.CrawlURI; import org.archive.modules.net.CrawlHost; import org.archive.modules.net.ServerCache; @@ -139,7 +138,7 @@ public class StatisticsTracker Checkpointable, BeanNameAware { @SuppressWarnings("unused") - private static final long serialVersionUID = 5L; + private static final long serialVersionUID = 6L; protected SeedModule seeds; public SeedModule getSeeds() { @@ -289,11 +288,6 @@ public class StatisticsTracker protected ConcurrentHashMap> sourceHostDistribution = new ConcurrentHashMap>(); - /* Keep track of 'top' hosts for live reports */ - protected TopNSet hostsDistributionTop; - protected TopNSet hostsBytesTop; - protected TopNSet hostsLastFinishedTop; - /** * Record of seeds and latest results */ @@ -350,10 +344,6 @@ public class StatisticsTracker this.processedSeedsRecords = bdb.getObjectCache("processedSeedsRecords", isRecover, SeedRecord.class); - this.hostsDistributionTop = new TopNSet(getLiveHostReportSize()); - this.hostsBytesTop = new TopNSet(getLiveHostReportSize()); - this.hostsLastFinishedTop = new TopNSet(getLiveHostReportSize()); - if(isRecover) { JSONObject json = recoveryCheckpoint.loadJson(beanName); @@ -363,19 +353,6 @@ public class StatisticsTracker crawlPauseStarted = json.getLong("crawlPauseStarted"); tallyCurrentPause(); - JSONUtils.putAllLongs( - hostsDistributionTop.getTopSet(), - json.getJSONObject("hostsDistributionTop")); - hostsDistributionTop.updateBounds(); - JSONUtils.putAllLongs( - hostsBytesTop.getTopSet(), - json.getJSONObject("hostsBytesTop")); - hostsBytesTop.updateBounds(); - JSONUtils.putAllLongs( - hostsLastFinishedTop.getTopSet(), - json.getJSONObject("hostsLastFinishedTop")); - hostsLastFinishedTop.updateBounds(); - JSONUtils.putAllAtomicLongs( mimeTypeDistribution, json.getJSONObject("mimeTypeDistribution")); @@ -758,11 +735,7 @@ public class StatisticsTracker incrementMapCount(mimeTypeDistribution, mime); incrementMapCount(mimeTypeBytes, mime, curi.getContentSize()); - // Save hosts stats. ServerCache sc = serverCache; - saveHostStats(sc.getHostFor(curi.getUURI()).getHostName(), - curi.getContentSize()); - if (getTrackSources() && curi.getData().containsKey(A_SOURCE_TAG)) { saveSourceStats((String)curi.getData().get(A_SOURCE_TAG), sc.getHostFor(curi.getUURI()). @@ -783,22 +756,6 @@ public class StatisticsTracker } - /** - * Update some running-stats based on a URI success - * - * @param hostname - * @param size - */ - protected void saveHostStats(String hostname, long size) { - // TODO: consider moving 'top' accounting elsewhere, such - // as the frontier or ServerCache itself - - CrawlHost host = serverCache.getHostFor(hostname); - hostsDistributionTop.update(hostname, host.getSubstats().getFetchSuccesses()); - hostsBytesTop.update(hostname, host.getSubstats().getSuccessBytes()); - hostsLastFinishedTop.update(hostname, host.getSubstats().getLastSuccessTime()); - } - public void crawledURINeedRetry(CrawlURI curi) { handleSeed(curi,"Failed to crawl seed, will retry"); } @@ -1074,10 +1031,6 @@ public class StatisticsTracker json.put("crawlPauseStarted",virtualCrawlPauseStarted); json.put("crawlTotalPausedTime",crawlTotalPausedTime); - json.put("hostsDistributionTop", hostsDistributionTop.getTopSet()); - json.put("hostsBytesTop", hostsBytesTop.getTopSet()); - json.put("hostsLastFinishedTop", hostsLastFinishedTop.getTopSet()); - json.put("mimeTypeDistribution", mimeTypeDistribution); json.put("mimeTypeBytes", mimeTypeBytes); json.put("statusCodeDistribution", statusCodeDistribution); From 96b927d9faf5c811a3c26773aaa8e247af4d3e87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristinn=20Sigur=C3=B0sson?= Date: Mon, 13 Jul 2015 14:55:26 +0000 Subject: [PATCH 2/5] Make sorting of host report configurable. Can now set a maximum size for sorted reports. Longer reports are unsorted. Default behavior is still to sort all reports. Default profile updated to reflect this new option in a commented-out section. --- .../crawler/reporting/HostsReport.java | 40 +++++++++++++++---- .../restlet/profile-crawler-beans.cxml | 4 +- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/reporting/HostsReport.java b/engine/src/main/java/org/archive/crawler/reporting/HostsReport.java index 4370e425..8206e3f3 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/HostsReport.java +++ b/engine/src/main/java/org/archive/crawler/reporting/HostsReport.java @@ -20,7 +20,7 @@ package org.archive.crawler.reporting; import java.io.PrintWriter; -import java.util.Map; +import java.util.Collection; import java.util.logging.Level; import java.util.logging.Logger; @@ -36,16 +36,40 @@ public class HostsReport extends Report { private final static Logger logger = Logger.getLogger(HostsReport.class.getName()); + + int maxSortSize = -1; + public int getMaxSortSize() { + return maxSortSize; + } + /** + * The maximum number of hosts allowed in a report while still sorting it. If the number of hosts exceeds + * this value, the generated report will not be sorted. A negative signifies no limit (always sort). + * A value of zero means never sort. Default -1, always sort. This matches the behavior before this + * parameter was introduced. + * + * This value can not be overridden by a sheet. + * + * @param maxSortSize + */ + public void setMaxSortSize(int maxSortSize) { + this.maxSortSize = maxSortSize; + } @Override public void write(final PrintWriter writer, StatisticsTracker stats) { - // TODO: only perform sorting on manageable number of hosts - DisposableStoredSortedMap hd = stats.calcReverseSortedHostsDistribution(); + Collection keys = null; + DisposableStoredSortedMap hd = null; + if (maxSortSize<0 || maxSortSize>stats.serverCache.hostKeys().size()) { + hd = stats.calcReverseSortedHostsDistribution(); + keys = hd.values(); + } else { + keys = stats.serverCache.hostKeys(); + } writer.print("[#urls] [#bytes] [host] [#robots] [#remaining] [#novel-urls] [#novel-bytes] [#dup-by-hash-urls] [#dup-by-hash-bytes] [#not-modified-urls] [#not-modified-bytes]\n"); - for (Map.Entry entry : hd.entrySet()) { + for (String key : keys) { // key is -count, value is hostname try { - CrawlHost host = stats.serverCache.getHostFor(entry.getValue()); + CrawlHost host = stats.serverCache.getHostFor(key); writeReportLine(writer, host.getSubstats().getFetchSuccesses(), host.getSubstats().getTotalBytes(), @@ -59,10 +83,12 @@ public class HostsReport extends Report { host.getSubstats().getNotModifiedUrls(), host.getSubstats().getNotModifiedBytes()); } catch (Exception e) { - logger.log(Level.WARNING, "unable to tally host stats for " + entry.getValue(), e); + logger.log(Level.WARNING, "unable to tally host stats for " + key, e); } } - hd.dispose(); + if (hd!=null) { + hd.dispose(); + } } protected void writeReportLine(PrintWriter writer, Object ... fields) { diff --git a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml index eb7d8550..319524a8 100644 --- a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml +++ b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml @@ -628,7 +628,9 @@ http://example.example/example - + + + From 0cf009eb4ccbbd8b5c778b4059217ef336b8582d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristinn=20Sigur=C3=B0sson?= Date: Mon, 13 Jul 2015 14:58:23 +0000 Subject: [PATCH 3/5] Use default value in sample profile. --- .../org/archive/crawler/restlet/profile-crawler-beans.cxml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml index 319524a8..f3e246ce 100644 --- a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml +++ b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml @@ -629,7 +629,7 @@ http://example.example/example - + From b15fbbd13e56325719bfdb57af375b0e752b17d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristinn=20Sigur=C3=B0sson?= Date: Mon, 13 Jul 2015 15:08:03 +0000 Subject: [PATCH 4/5] Enable suppression of empty hosts in report. Until now hosts that have never been crawled are included in the hosts report. This can be a very large number of hosts and is infrequently of interest. This makes it possible to exclude the empty hosts from the report. Default behavior remains to include them. The option is highlighted by including it in the relevant, commented-out section of the default profile. --- .../crawler/reporting/HostsReport.java | 52 +++++++++++++------ .../restlet/profile-crawler-beans.cxml | 1 + 2 files changed, 38 insertions(+), 15 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/reporting/HostsReport.java b/engine/src/main/java/org/archive/crawler/reporting/HostsReport.java index 8206e3f3..6a3175dd 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/HostsReport.java +++ b/engine/src/main/java/org/archive/crawler/reporting/HostsReport.java @@ -47,15 +47,34 @@ public class HostsReport extends Report { * A value of zero means never sort. Default -1, always sort. This matches the behavior before this * parameter was introduced. * - * This value can not be overridden by a sheet. + * This value can not be overridden by a sheet. It may be safely edited at runtime. * * @param maxSortSize */ public void setMaxSortSize(int maxSortSize) { this.maxSortSize = maxSortSize; } - - @Override + + boolean suppressEmptyHosts = false; + public boolean isSuppressEmptyHosts() { + return suppressEmptyHosts; + } + /** + * If true, hosts for whom no URLs have been fetched will be suppressed in this report. + * Such hosts are recorded when the crawler encounters an URL for a host but has not yet (and may never) + * processed any URL for the host. This can happen for many reason's, related to scoping and queue budgeting + * among others. + * Default behavior is to include these non-crawled hosts. + * + * This value can no t be overridden by a sheet. It may be safely edited at runtime. + * + * @param suppressEmptyHosts + */ + public void setSuppressEmptyHosts(boolean suppressEmptyHosts) { + this.suppressEmptyHosts = suppressEmptyHosts; + } + + @Override public void write(final PrintWriter writer, StatisticsTracker stats) { Collection keys = null; DisposableStoredSortedMap hd = null; @@ -70,18 +89,21 @@ public class HostsReport extends Report { // key is -count, value is hostname try { CrawlHost host = stats.serverCache.getHostFor(key); - writeReportLine(writer, - host.getSubstats().getFetchSuccesses(), - host.getSubstats().getTotalBytes(), - host.fixUpName(), - host.getSubstats().getRobotsDenials(), - host.getSubstats().getRemaining(), - host.getSubstats().getNovelUrls(), - host.getSubstats().getNovelBytes(), - host.getSubstats().getDupByHashUrls(), - host.getSubstats().getDupByHashBytes(), - host.getSubstats().getNotModifiedUrls(), - host.getSubstats().getNotModifiedBytes()); + long fetchSuccesses = host.getSubstats().getFetchSuccesses(); + if (!suppressEmptyHosts || fetchSuccesses>0) { + writeReportLine(writer, + fetchSuccesses, + host.getSubstats().getTotalBytes(), + host.fixUpName(), + host.getSubstats().getRobotsDenials(), + host.getSubstats().getRemaining(), + host.getSubstats().getNovelUrls(), + host.getSubstats().getNovelBytes(), + host.getSubstats().getDupByHashUrls(), + host.getSubstats().getDupByHashBytes(), + host.getSubstats().getNotModifiedUrls(), + host.getSubstats().getNotModifiedBytes()); + } } catch (Exception e) { logger.log(Level.WARNING, "unable to tally host stats for " + key, e); } diff --git a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml index f3e246ce..589646e4 100644 --- a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml +++ b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml @@ -630,6 +630,7 @@ http://example.example/example + From 3495851029c1911dd03998e619fe80a51fc9a526 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristinn=20Sigur=C3=B0sson?= Date: Mon, 13 Jul 2015 15:10:44 +0000 Subject: [PATCH 5/5] JavaDoc typo --- .../main/java/org/archive/crawler/reporting/HostsReport.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/src/main/java/org/archive/crawler/reporting/HostsReport.java b/engine/src/main/java/org/archive/crawler/reporting/HostsReport.java index 6a3175dd..a781c1aa 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/HostsReport.java +++ b/engine/src/main/java/org/archive/crawler/reporting/HostsReport.java @@ -66,7 +66,7 @@ public class HostsReport extends Report { * among others. * Default behavior is to include these non-crawled hosts. * - * This value can no t be overridden by a sheet. It may be safely edited at runtime. + * This value can not be overridden by a sheet. It may be safely edited at runtime. * * @param suppressEmptyHosts */