* HostsReport.java

url-encode hosts (other than "dns:") in host report, in case of hostname
    containing whitespace or other unusual characters, which can happen
This commit is contained in:
nlevitt committed 2011-03-01 23:43:58 +00:00
1 parent 3fc3d324e8
commit 17b12ca19b
1 file changed
+16 -2
@@ -20,6 +20,8 @@
package org.archive.crawler.reporting;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.Map;
import org.apache.commons.collections.Closure;
@@ -33,6 +35,18 @@ import org.archive.modules.net.CrawlHost;
*/
public class HostsReport extends Report {
protected String fixup(String hostName) {
if ("dns:".equals(hostName)) {
return hostName;
} else {
try {
return URLEncoder.encode(hostName, "UTF-8");
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
}
}
@Override
public void write(final PrintWriter writer) {
// TODO: use CrawlHosts for all stats; only perform sorting on
@@ -47,7 +61,7 @@ public class HostsReport extends Report {
writeReportLine(writer,
count,
stats.getBytesPerHost(entry.getValue()),
entry.getValue(),
fixup(entry.getValue()),
host.getSubstats().getRobotsDenials(),
host.getSubstats().getRemaining(),
host.getSubstats().getNovelUrls(),
@@ -67,7 +81,7 @@ public class HostsReport extends Report {
writeReportLine(writer,
host.getSubstats().getRecordedFinishes(),
host.getSubstats().getTotalBytes(),
host.getHostName(),
fixup(host.getHostName()),
host.getSubstats().getRobotsDenials(),
host.getSubstats().getRemaining(),
host.getSubstats().getNovelUrls(),