lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-07-16 12:37:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

part of fix for HER-1736 long pause at startup reading large per-job job.log(s)

Browse Source 
* CrawlJob.java
    added boolean partialJobLogScan, and print "at least X launches" when true
* JobResource.java
    print "at least X launches" when partialJobLogScan
This commit is contained in:
szznax
2010-02-12 23:21:59 +00:00
parent 18d459f6b9
commit 2d441d611e
2 changed files with 13 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/src/main/java/org/archive/crawler/framework/CrawlJob.java
+10 -2
View File
@@ -88,6 +88,7 @@ public class CrawlJob implements Comparable<CrawlJob>, ApplicationListener {
File primaryConfig;
PathSharingContext ac;
int launchCount;
public boolean partialJobLogScan;
DateTime lastLaunch;
AlertThreadGroup alertThreadGroup;
@@ -96,6 +97,7 @@ public class CrawlJob implements Comparable<CrawlJob>, ApplicationListener {
public CrawlJob(File cxml) {
primaryConfig = cxml;
partialJobLogScan = false;
scanJobLog();
alertThreadGroup = new AlertThreadGroup(getShortName());
}
@@ -159,8 +161,8 @@ public class CrawlJob implements Comparable<CrawlJob>, ApplicationListener {
try {
Pattern launchLine = Pattern.compile("(\\S+) (\\S+) Job launched");
if (jobLog.length() > FileUtils.ONE_KB * 100) {
partialJobLogScan = true;
// indicate that job log is too big to count launches accurately
launchCount = -1;
getJobLogger().log(Level.INFO,"Job log (" + jobLog.toString()
+ ") too big (" + NumberFormat.getInstance().format(jobLog.length())
+ " bytes) to efficiently count launches. OK to move aside.");
@@ -172,6 +174,7 @@ public class CrawlJob implements Comparable<CrawlJob>, ApplicationListener {
while ((line = jobLogBfr.readLine()) != null) {
Matcher m = launchLine.matcher(line);
if (m.matches()) {
launchCount++;
lastLaunch = new DateTime(m.group(1));
}
}
@@ -223,7 +226,12 @@ public class CrawlJob implements Comparable<CrawlJob>, ApplicationListener {
if(isProfile()) {
pw.println("(profile)");
}
pw.println(" " + getLaunchCount() + " launches");
if (true == partialJobLogScan) {
pw.print(" at least ");
} else {
pw.print(" ");
}
pw.println(getLaunchCount() + " launches");
pw.println("<br/><span style='color:#666'>");
pw.println(getPrimaryConfig());
pw.println("</span><br/>");
engine/src/main/java/org/archive/crawler/restlet/JobResource.java
+3
View File
@@ -233,6 +233,9 @@ public class JobResource extends BaseResource {
pw.println("</head><body>");
pw.print("<h1>Job <i>"+cj.getShortName()+"</i> (");
if (cj.partialJobLogScan) {
pw.print(" at least ");
}
pw.print(cj.getLaunchCount() + " launches");
if(cj.getLastLaunch()!=null) {
long ago = System.currentTimeMillis() - cj.getLastLaunch().getMillis();