From acfe95fea3ada0f80521112331379239300cd17f Mon Sep 17 00:00:00 2001 From: gojomo Date: Sat, 13 Feb 2010 00:47:41 +0000 Subject: [PATCH] naming/branching/visibility review for HER-1736 long pause at startup reading large per-job job.log(s) * CrawlJob.java added accessor, renamed a bit, unified 2 branches * JobResource.java use accessor --- .../archive/crawler/framework/CrawlJob.java | 51 ++++++++----------- .../archive/crawler/restlet/JobResource.java | 12 ++--- 2 files changed, 27 insertions(+), 36 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java b/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java index 7af51246..cceb6d6b 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java +++ b/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java @@ -48,7 +48,6 @@ import javax.xml.parsers.ParserConfigurationException; import org.apache.commons.collections.ListUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.io.FileUtils; -import org.apache.commons.io.LineIterator; import org.apache.commons.lang.StringUtils; import org.archive.crawler.event.CrawlStateEvent; import org.archive.crawler.reporting.AlertThreadGroup; @@ -88,7 +87,7 @@ public class CrawlJob implements Comparable, ApplicationListener { File primaryConfig; PathSharingContext ac; int launchCount; - public boolean partialJobLogScan; + boolean isLaunchInfoPartial; DateTime lastLaunch; AlertThreadGroup alertThreadGroup; @@ -97,7 +96,7 @@ public class CrawlJob implements Comparable, ApplicationListener { public CrawlJob(File cxml) { primaryConfig = cxml; - partialJobLogScan = false; + isLaunchInfoPartial = false; scanJobLog(); alertThreadGroup = new AlertThreadGroup(getShortName()); } @@ -119,6 +118,9 @@ public class CrawlJob implements Comparable, ApplicationListener { return ac; } + public boolean isLaunchInfoPartial() { + return isLaunchInfoPartial; + } /** * Get a logger to a distinguished file, job.log in the job's @@ -160,37 +162,27 @@ public class CrawlJob implements Comparable, ApplicationListener { try { Pattern launchLine = Pattern.compile("(\\S+) (\\S+) Job launched"); + long startPosition = 0; if (jobLog.length() > FileUtils.ONE_KB * 100) { - partialJobLogScan = true; + isLaunchInfoPartial = true; + startPosition = jobLog.length()-(FileUtils.ONE_KB * 100); // indicate that job log is too big to count launches accurately getJobLogger().log(Level.INFO,"Job log (" + jobLog.toString() + ") too big (" + NumberFormat.getInstance().format(jobLog.length()) + " bytes) to efficiently count launches. OK to move aside."); - // attempt to find last launch near end of file - FileInputStream jobLogIn = new FileInputStream(jobLog); - jobLogIn.getChannel().position(jobLog.length()-(FileUtils.ONE_KB * 100)); - BufferedReader jobLogBfr = new BufferedReader(new InputStreamReader(jobLogIn)); - String line; - while ((line = jobLogBfr.readLine()) != null) { - Matcher m = launchLine.matcher(line); - if (m.matches()) { - launchCount++; - lastLaunch = new DateTime(m.group(1)); - } - } - } else { - // scan the entire job log - LineIterator lines = FileUtils.lineIterator(jobLog); - while(lines.hasNext()) { - String line = lines.nextLine(); - Matcher m = launchLine.matcher(line); - if(m.matches()) { - launchCount++; - lastLaunch = new DateTime(m.group(1)); - } - } - LineIterator.closeQuietly(lines); } + FileInputStream jobLogIn = new FileInputStream(jobLog); + jobLogIn.getChannel().position(startPosition); + BufferedReader jobLogReader = new BufferedReader(new InputStreamReader(jobLogIn)); + String line; + while ((line = jobLogReader.readLine()) != null) { + Matcher m = launchLine.matcher(line); + if (m.matches()) { + launchCount++; + lastLaunch = new DateTime(m.group(1)); + } + } + jobLogReader.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); @@ -226,7 +218,7 @@ public class CrawlJob implements Comparable, ApplicationListener { if(isProfile()) { pw.println("(profile)"); } - if (true == partialJobLogScan) { + if (true == isLaunchInfoPartial) { pw.print(" at least "); } else { pw.print(" "); @@ -899,5 +891,4 @@ public class CrawlJob implements Comparable, ApplicationListener { return null; } } - }//EOC diff --git a/engine/src/main/java/org/archive/crawler/restlet/JobResource.java b/engine/src/main/java/org/archive/crawler/restlet/JobResource.java index 9a3bca13..af69fd08 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/JobResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/JobResource.java @@ -85,7 +85,7 @@ public class JobResource extends BaseResource { } public Representation represent(Variant variant) throws ResourceException { - if (cj == null) { + if(cj==null) { throw new ResourceException(404); } @@ -94,7 +94,7 @@ public class JobResource extends BaseResource { representation = new WriterRepresentation(MediaType.APPLICATION_XML) { public void write(Writer writer) throws IOException { XmlMarshaller.marshalDocument(writer, "job", makePresentableMap()); - } + } }; } else { representation = new WriterRepresentation(MediaType.TEXT_HTML) { @@ -233,7 +233,7 @@ public class JobResource extends BaseResource { pw.println(""); pw.print("

Job "+cj.getShortName()+" ("); - if (cj.partialJobLogScan) { + if (cj.isLaunchInfoPartial()) { pw.print(" at least "); } pw.print(cj.getLaunchCount() + " launches"); @@ -556,7 +556,7 @@ public class JobResource extends BaseResource { // copy op? Form form = null; - form = getRequest().getEntityAsForm(); + form = getRequest().getEntityAsForm(); String copyTo = form.getFirstValue("copyTo"); if(copyTo!=null) { copyJob(copyTo,"on".equals(form.getFirstValue("asProfile"))); @@ -596,10 +596,10 @@ public class JobResource extends BaseResource { cj.terminate(); } AlertThreadGroup.setThreadLogger(null); - + // default: redirect to GET self getResponse().redirectSeeOther(getRequest().getOriginalRef()); - } + } protected void copyJob(String copyTo, boolean asProfile) throws ResourceException { try {