diff --git a/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java b/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java index f5d73dae..44e89c64 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java +++ b/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java @@ -576,7 +576,11 @@ public class CrawlJob implements Comparable, ApplicationListener{ String filePath = f.getCanonicalPath(); String jobPath = getJobDir().getCanonicalPath(); if(filePath.startsWith(jobPath)) { - return filePath.substring(jobPath.length()).replace(File.separatorChar, '/'); + String jobRelative = filePath.substring(jobPath.length()).replace(File.separatorChar, '/'); + if(jobRelative.startsWith("/")) { + jobRelative = jobRelative.substring(1); + } + return jobRelative; } } catch (IOException e) { getJobLogger().log(Level.WARNING,"bad file: "+f); diff --git a/engine/src/main/java/org/archive/crawler/restlet/EngineApplication.java b/engine/src/main/java/org/archive/crawler/restlet/EngineApplication.java index 437ca314..d779873f 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EngineApplication.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EngineApplication.java @@ -80,7 +80,8 @@ public class EngineApplication extends Application { jobdir.setEditFilter(JobResource.EDIT_FILTER); router.attach("/engine/job/{job}/jobdir",jobdir); router.attach("/engine/job/{job}",JobResource.class); - + router.attach("/engine/job/{job}/report/{reportClass}",ReportGenResource.class); + return router; } diff --git a/engine/src/main/java/org/archive/crawler/restlet/JobResource.java b/engine/src/main/java/org/archive/crawler/restlet/JobResource.java index d2b9df6a..c494778b 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/JobResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/JobResource.java @@ -31,6 +31,8 @@ import org.apache.commons.io.filefilter.IOFileFilter; import org.apache.commons.lang.StringEscapeUtils; import org.archive.crawler.framework.CrawlJob; import org.archive.crawler.framework.Engine; +import org.archive.crawler.reporting.Report; +import org.archive.crawler.reporting.StatisticsTracker; import org.archive.spring.ConfigPath; import org.archive.util.ArchiveUtils; import org.archive.util.FileUtils; @@ -97,10 +99,17 @@ public class JobResource extends Resource { pw.print(cj.getLaunchCount() + " launches"); if(cj.getLastLaunch()!=null) { long ago = System.currentTimeMillis() - cj.getLastLaunch().getMillis(); - pw.println(", last "+ArchiveUtils.formatMillisecondsToConventional(ago, 2)+" ago)"); + pw.println(", last "+ArchiveUtils.formatMillisecondsToConventional(ago, 2)+" ago"); } pw.println(")"); + if(cj.isProfile()) { + pw.print( + "

As a profile, this job may be built for " + + "testing purposes but not launched. Use the 'copy job to' " + + "functionality at bottom to copy this profile to a " + + "launchable job.

"); + } // button controls pw.println("
"); @@ -213,7 +222,7 @@ public class JobResource extends Resource { pw.println("
Alerts
  "); pw.println(cj.getAlertCount()==0 ? "none" : cj.getAlertCount()); if(cj.getAlertCount()>0) { - pw.println("tail alert log..."); @@ -228,10 +237,10 @@ public class JobResource extends Resource { pw.println("
Elapsed
  "); pw.println(cj.elapsedReport()); - pw.println("
Threads
  "); + pw.println("
Threads
  "); pw.println(cj.threadReport()); - pw.println("
Frontier
  "); + pw.println("
Frontier
  "); pw.println(cj.frontierReport()); pw.println("
Memory
  "); @@ -240,7 +249,7 @@ public class JobResource extends Resource { if(cj.isRunning() || (cj.isContainerOk() && !cj.isLaunchable())) { // show crawl log for running or finished crawls pw.println("

Crawl Log"); - pw.println("(more)"); @@ -266,6 +275,15 @@ public class JobResource extends Resource { } + if(cj.isContainerOk()) { + pw.println("

Reports

"); + for(Class reportClass : StatisticsTracker.LIVE_REPORTS) { + String className = reportClass.getSimpleName(); + String shortName = className.substring(0,className.length()-"Report".length()); + pw.println(""+shortName+""); + } + } + pw.println("

Files

"); pw.println("

Browse Job Directory

"); // specific paths from wired context @@ -305,11 +323,11 @@ public class JobResource extends Resource { pw.println(f); return; } - pw.println("" + f +""); if(EDIT_FILTER.accept(f)) { - pw.println("[edit]
"); } diff --git a/engine/src/main/java/org/archive/crawler/restlet/ReportGenResource.java b/engine/src/main/java/org/archive/crawler/restlet/ReportGenResource.java new file mode 100644 index 00000000..224c722f --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/restlet/ReportGenResource.java @@ -0,0 +1,77 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.restlet; + +import java.io.File; + +import org.archive.crawler.framework.CrawlJob; +import org.archive.crawler.framework.Engine; +import org.restlet.Context; +import org.restlet.data.MediaType; +import org.restlet.data.Request; +import org.restlet.data.Response; +import org.restlet.resource.Representation; +import org.restlet.resource.Resource; +import org.restlet.resource.ResourceException; +import org.restlet.resource.StringRepresentation; +import org.restlet.resource.Variant; + +/** + * Restlet Resource which generates fresh reports and then redirects + * requests to the report in the filesystem. + * + * @contributor gojomo + */ +public class ReportGenResource extends Resource { + CrawlJob cj; + String reportClass; + + public ReportGenResource(Context ctx, Request req, Response res) throws ResourceException { + super(ctx, req, res); + getVariants().add(new Variant(MediaType.TEXT_PLAIN)); + cj = getEngine().getJob((String)req.getAttributes().get("job")); + if(cj==null) { + throw new ResourceException(404); + } + reportClass = (String)req.getAttributes().get("reportClass"); + } + + public Representation represent(Variant variant) throws ResourceException { + // generate report + File f = cj.getCrawlController().getStatisticsTracker().writeReportFile(reportClass); + if (f==null) { + throw new ResourceException(500); + } + // redirect + String relative = cj.jobDirRelativePath(f); + if(relative!=null) { + getResponse().redirectSeeOther("../jobdir/"+relative); + return new StringRepresentation(""); + } else { + return new StringRepresentation( + "Report dumped to "+f.getAbsolutePath() + +" (outside job directory)"); + } + } + + protected Engine getEngine() { + return ((EngineApplication)getApplication()).getEngine(); + } +}