From bba186f3336bcd016a413f6544d384d719f51e0f Mon Sep 17 00:00:00 2001
From: gojomo <gojomo@daa5b2f2-a927-0410-8b2d-f5f262fa301a>
Date: Fri, 9 Jul 2010 22:39:22 +0000
Subject: [PATCH] [HER-1799] H3: expand 'inferred' URI options to optionally
 include root-page (/) * ExtractorHTTP.java     add property inferRootPage; if
 true (not the default), always consider '/' inferred from all HTTP(S) URIs *
 Hop.java, LinkContext.java     add new hop-type 'I' for inferred URIs; add
 corresponding LinkContext * ExtractorImpliedURI.java     use new 'I'nferred
 types

---
 .../archive/modules/extractor/ExtractorHTTP.java  | 15 ++++++++++++++-
 .../modules/extractor/ExtractorImpliedURI.java    |  4 ++--
 .../java/org/archive/modules/extractor/Hop.java   |  7 ++++++-
 .../archive/modules/extractor/LinkContext.java    |  9 ++++++++-
 4 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java
index f53b0a21..2b321543 100644
--- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java
+++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java
@@ -39,6 +39,16 @@ public class ExtractorHTTP extends Extractor {
     public ExtractorHTTP() {
     }
 
+    /** should all HTTP URIs be used to infer a link to the site's root? */
+    boolean inferRootPage = false; 
+    public boolean getInferRootPage() {
+        return inferRootPage;
+    }
+    public void setInferRootPage(boolean inferRootPage) {
+        this.inferRootPage = inferRootPage;
+    }
+
+
     @Override
     protected boolean shouldProcess(CrawlURI uri) {
         if (uri.getFetchStatus() <= 0) {
@@ -57,7 +67,10 @@ public class ExtractorHTTP extends Extractor {
         addHeaderLink(curi, method.getResponseHeader("Content-Location"));
         
         // try /favicon.ico for every HTTP(S) URI
-        addOutlink(curi, "/favicon.ico", LinkContext.EMBED_MISC, Hop.EMBED);
+        addOutlink(curi, "/favicon.ico", LinkContext.INFERRED_MISC, Hop.INFERRED);
+        if(getInferRootPage()) {
+            addOutlink(curi, "/", LinkContext.INFERRED_MISC, Hop.INFERRED);
+        }
     }
 
     protected void addHeaderLink(CrawlURI curi, Header loc) {
diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorImpliedURI.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorImpliedURI.java
index 95763c55..f8dc2777 100644
--- a/modules/src/main/java/org/archive/modules/extractor/ExtractorImpliedURI.java
+++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorImpliedURI.java
@@ -132,8 +132,8 @@ public class ExtractorImpliedURI extends Extractor {
                 try {
                     UURI src = curi.getUURI();
                     UURI target = UURIFactory.getInstance(implied);
-                    LinkContext lc = LinkContext.SPECULATIVE_MISC;
-                    Hop hop = Hop.SPECULATIVE;
+                    LinkContext lc = LinkContext.INFERRED_MISC;
+                    Hop hop = Hop.INFERRED;
                     Link out = new Link(src, target, lc, hop);
                     curi.getOutLinks().add(out);
                     numberOfLinksExtracted.incrementAndGet();
diff --git a/modules/src/main/java/org/archive/modules/extractor/Hop.java b/modules/src/main/java/org/archive/modules/extractor/Hop.java
index 775bdb97..5b97490e 100644
--- a/modules/src/main/java/org/archive/modules/extractor/Hop.java
+++ b/modules/src/main/java/org/archive/modules/extractor/Hop.java
@@ -48,8 +48,13 @@ public enum Hop {
     /** 
      * Referral/redirect links, like header 'Location:' on a 301/302 response. 
      */
-    REFER('R');
+    REFER('R'),
 
+    /** 
+     * Inferred/implied links -- not necessarily literally in the source 
+     * material, but deduced by convention.
+     */
+    INFERRED('I');
     
     /** The hop character for logs. */
     private char hopChar;
diff --git a/modules/src/main/java/org/archive/modules/extractor/LinkContext.java b/modules/src/main/java/org/archive/modules/extractor/LinkContext.java
index 6fc89db4..d09dbd80 100644
--- a/modules/src/main/java/org/archive/modules/extractor/LinkContext.java
+++ b/modules/src/main/java/org/archive/modules/extractor/LinkContext.java
@@ -28,8 +28,9 @@ import java.io.Serializable;
  * @author pjack
  */
 public abstract class LinkContext implements Serializable {
+    private static final long serialVersionUID = 1L;
+
 
-    
     /** Class for representing handy default LinkContext values. */
     private static class SimpleLinkContext extends LinkContext {
 
@@ -65,6 +66,12 @@ public abstract class LinkContext implements Serializable {
     final public static LinkContext SPECULATIVE_MISC
      = new SimpleLinkContext("=SPECULATIVE_MISC");
 
+    /** 
+     * Stand-in value for inferred urls without  other context. 
+     */
+    final public static LinkContext INFERRED_MISC
+        = new SimpleLinkContext("=INFERRED_MISC");
+    
     /** Stand-in value for prerequisite urls without other context. */
     final public static LinkContext PREREQ_MISC
      = new SimpleLinkContext("=PREREQ_MISC");