[HER-1598] Restletify: enable report generating, viewing, refreshing

* ReportGenResource.java
    restlet to generate-report-to-file and redirect to view file
* JobResource.java
    links to reports from appropriate places for built job
    clarify profiles, simplify into-job-dir path-construction
* CrawlJob.java
    remove leading '/' from job-relative paths
* EngineApplication.java
    install ReportGenResource at appropriate in-job path

engine/src/main/java/org/archive/crawler/framework/CrawlJob.java

@@ -576,7 +576,11 @@ public class CrawlJob implements Comparable<CrawlJob>, ApplicationListener{
             String filePath = f.getCanonicalPath();
             String jobPath = getJobDir().getCanonicalPath();
             if(filePath.startsWith(jobPath)) {
-                return filePath.substring(jobPath.length()).replace(File.separatorChar, '/');
+                String jobRelative = filePath.substring(jobPath.length()).replace(File.separatorChar, '/');
+                if(jobRelative.startsWith("/")) {
+                    jobRelative = jobRelative.substring(1); 
+                }
+                return jobRelative;
             }
         } catch (IOException e) {
             getJobLogger().log(Level.WARNING,"bad file: "+f);

engine/src/main/java/org/archive/crawler/restlet/EngineApplication.java

@@ -80,7 +80,8 @@ public class EngineApplication extends Application {
         jobdir.setEditFilter(JobResource.EDIT_FILTER);
         router.attach("/engine/job/{job}/jobdir",jobdir);
         router.attach("/engine/job/{job}",JobResource.class);
-        
+        router.attach("/engine/job/{job}/report/{reportClass}",ReportGenResource.class);
+
         return router;
     }
 

engine/src/main/java/org/archive/crawler/restlet/JobResource.java

@@ -31,6 +31,8 @@ import org.apache.commons.io.filefilter.IOFileFilter;
 import org.apache.commons.lang.StringEscapeUtils;
 import org.archive.crawler.framework.CrawlJob;
 import org.archive.crawler.framework.Engine;
+import org.archive.crawler.reporting.Report;
+import org.archive.crawler.reporting.StatisticsTracker;
 import org.archive.spring.ConfigPath;
 import org.archive.util.ArchiveUtils;
 import org.archive.util.FileUtils;
@@ -97,10 +99,17 @@ public class JobResource extends Resource {
         pw.print(cj.getLaunchCount() + " launches");
         if(cj.getLastLaunch()!=null) {
             long ago = System.currentTimeMillis() - cj.getLastLaunch().getMillis();
-            pw.println(", last "+ArchiveUtils.formatMillisecondsToConventional(ago, 2)+" ago)");
+            pw.println(", last "+ArchiveUtils.formatMillisecondsToConventional(ago, 2)+" ago");
         }
         pw.println(")</h1>");
         
+        if(cj.isProfile()) {
+            pw.print(
+                "<p>As a <i>profile</i>, this job may be built for " +
+                "testing purposes but not launched. Use the 'copy job to' " +
+                "functionality at bottom to copy this profile to a " +
+                "launchable job.</p>");
+        }
         
         // button controls
         pw.println("<form method='POST'>");
@@ -213,7 +222,7 @@ public class JobResource extends Resource {
             pw.println("<br/><b>Alerts</b><br>&nbsp;&nbsp;");
             pw.println(cj.getAlertCount()==0 ? "<i>none</i>" : cj.getAlertCount()); 
             if(cj.getAlertCount()>0) {
-                pw.println("<a href='jobdir"
+                pw.println("<a href='jobdir/"
                         +cj.jobDirRelativePath(
                                 cj.getCrawlController().getLoggerModule().getAlertsLogPath().getFile())
                         +"?format=paged&pos=-1&lines=-128'>tail alert log...</a>");
@@ -228,10 +237,10 @@ public class JobResource extends Resource {
             pw.println("<br/><b>Elapsed</b><br/>&nbsp;&nbsp;");
             pw.println(cj.elapsedReport());
             
-            pw.println("<br/><b>Threads</b><br/>&nbsp;&nbsp;");
+            pw.println("<br/><a href='report/ToeThreadsReport'><b>Threads</b></a><br/>&nbsp;&nbsp;");
             pw.println(cj.threadReport());
     
-            pw.println("<br/><b>Frontier</b><br/>&nbsp;&nbsp;");
+            pw.println("<br/><a href='report/FrontierSummaryReport'><b>Frontier</b></a><br/>&nbsp;&nbsp;");
             pw.println(cj.frontierReport());
             
             pw.println("<br/><b>Memory</b><br/>&nbsp;&nbsp;");
@@ -240,7 +249,7 @@ public class JobResource extends Resource {
             if(cj.isRunning() || (cj.isContainerOk() && !cj.isLaunchable())) {
                 // show crawl log for running or finished crawls
                 pw.println("<h3>Crawl Log");
-                pw.println("(<a href='jobdir"
+                pw.println("(<a href='jobdir/"
                         +cj.jobDirRelativePath(
                                 cj.getCrawlController().getLoggerModule().getCrawlLogPath().getFile())
                         +"?format=paged&pos=-1&lines=-128&reverse=y'><i>more</i></a>)");
@@ -266,6 +275,15 @@ public class JobResource extends Resource {
             
         }
         
+        if(cj.isContainerOk()) {
+            pw.println("<h2>Reports</h2>");
+            for(Class<Report> reportClass : StatisticsTracker.LIVE_REPORTS) {
+                String className = reportClass.getSimpleName();
+                String shortName = className.substring(0,className.length()-"Report".length());
+                pw.println("<a href='report/"+className+"'>"+shortName+"</a>");
+            }
+        }
+        
         pw.println("<h2>Files</h2>");
         pw.println("<h3>Browse <a href='jobdir'>Job Directory</a></h3>");
         // specific paths from wired context
@@ -305,11 +323,11 @@ public class JobResource extends Resource {
             pw.println(f);
             return;
         }
-        pw.println("<a href='jobdir" 
+        pw.println("<a href='jobdir/" 
                 + jobDirRelative + "'>" 
                 + f +"</a>");
         if(EDIT_FILTER.accept(f)) {
-            pw.println("[<a href='jobdir" 
+            pw.println("[<a href='jobdir/" 
                     + jobDirRelative 
                     +  "?format=textedit'>edit</a>]<br/>");
         }

engine/src/main/java/org/archive/crawler/restlet/ReportGenResource.java

@@ -0,0 +1,77 @@
+/*
+ *  This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.archive.crawler.restlet;
+
+import java.io.File;
+
+import org.archive.crawler.framework.CrawlJob;
+import org.archive.crawler.framework.Engine;
+import org.restlet.Context;
+import org.restlet.data.MediaType;
+import org.restlet.data.Request;
+import org.restlet.data.Response;
+import org.restlet.resource.Representation;
+import org.restlet.resource.Resource;
+import org.restlet.resource.ResourceException;
+import org.restlet.resource.StringRepresentation;
+import org.restlet.resource.Variant;
+
+/**
+ * Restlet Resource which generates fresh reports and then redirects
+ * requests to the report in the filesystem. 
+ * 
+ * @contributor gojomo
+ */
+public class ReportGenResource extends Resource {
+    CrawlJob cj; 
+    String reportClass;
+    
+    public ReportGenResource(Context ctx, Request req, Response res) throws ResourceException {
+        super(ctx, req, res);
+        getVariants().add(new Variant(MediaType.TEXT_PLAIN));
+        cj = getEngine().getJob((String)req.getAttributes().get("job"));
+        if(cj==null) {
+            throw new ResourceException(404);
+        }
+        reportClass = (String)req.getAttributes().get("reportClass");
+    }
+
+    public Representation represent(Variant variant) throws ResourceException {
+        // generate report
+        File f = cj.getCrawlController().getStatisticsTracker().writeReportFile(reportClass); 
+        if (f==null) {
+            throw new ResourceException(500);
+        }
+        // redirect
+        String relative = cj.jobDirRelativePath(f);
+        if(relative!=null) {
+            getResponse().redirectSeeOther("../jobdir/"+relative);
+            return new StringRepresentation("");
+        } else {
+            return new StringRepresentation(
+                    "Report dumped to "+f.getAbsolutePath()
+                    +" (outside job directory)");
+        }
+    }
+    
+    protected Engine getEngine() {
+        return ((EngineApplication)getApplication()).getEngine();
+    }
+}