mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-09 23:43:29 +00:00
* HostsReport.java
url-encode hosts (other than "dns:") in host report, in case of hostname
containing whitespace or other unusual characters, which can happen
This commit is contained in:
1 parent
3fc3d324e8
commit
17b12ca19b
1 file changed
+16
-2
@@ -20,6 +20,8 @@
|
||||
package org.archive.crawler.reporting;
|
||||
|
||||
import java.io.PrintWriter;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.net.URLEncoder;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.commons.collections.Closure;
|
||||
@@ -33,6 +35,18 @@ import org.archive.modules.net.CrawlHost;
|
||||
*/
|
||||
public class HostsReport extends Report {
|
||||
|
||||
protected String fixup(String hostName) {
|
||||
if ("dns:".equals(hostName)) {
|
||||
return hostName;
|
||||
} else {
|
||||
try {
|
||||
return URLEncoder.encode(hostName, "UTF-8");
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(final PrintWriter writer) {
|
||||
// TODO: use CrawlHosts for all stats; only perform sorting on
|
||||
@@ -47,7 +61,7 @@ public class HostsReport extends Report {
|
||||
writeReportLine(writer,
|
||||
count,
|
||||
stats.getBytesPerHost(entry.getValue()),
|
||||
entry.getValue(),
|
||||
fixup(entry.getValue()),
|
||||
host.getSubstats().getRobotsDenials(),
|
||||
host.getSubstats().getRemaining(),
|
||||
host.getSubstats().getNovelUrls(),
|
||||
@@ -67,7 +81,7 @@ public class HostsReport extends Report {
|
||||
writeReportLine(writer,
|
||||
host.getSubstats().getRecordedFinishes(),
|
||||
host.getSubstats().getTotalBytes(),
|
||||
host.getHostName(),
|
||||
fixup(host.getHostName()),
|
||||
host.getSubstats().getRobotsDenials(),
|
||||
host.getSubstats().getRemaining(),
|
||||
host.getSubstats().getNovelUrls(),
|
||||
|
||||
Reference in new issue
Block a user