From 3e35c967abdebd464c8df19dcb1c6862e973d0ef Mon Sep 17 00:00:00 2001
From: Noah Levitt <nlevitt@archive.org>
Date: Mon, 16 Jun 2014 18:33:25 -0700
Subject: [PATCH 1/3] upgrade hbase to cdh5

---
 contrib/pom.xml                               | 27 ++-----------------
 .../crawler/frontier/AMQPUrlReceiver.java     |  2 +-
 .../archive/modules/recrawl/hbase/HBase.java  |  7 +++--
 .../hbase/HBaseContentDigestHistory.java      |  5 ++--
 .../recrawl/hbase/SingleHBaseTable.java       |  6 -----
 5 files changed, 9 insertions(+), 38 deletions(-)

diff --git a/contrib/pom.xml b/contrib/pom.xml
index a9d16633..23e02519 100644
--- a/contrib/pom.xml
+++ b/contrib/pom.xml
@@ -16,31 +16,8 @@
 	<dependencies>
 		<dependency>
 			<groupId>org.apache.hbase</groupId>
-			<artifactId>hbase</artifactId>
-			<version>0.90.6-cdh3u5</version>
-			<scope>compile</scope>
-			<exclusions>
-				<exclusion>
-					<groupId>org.mortbay.jetty</groupId>
-					<artifactId>servlet-api-2.5</artifactId>
-				</exclusion>
-				<exclusion>
-					<groupId>org.mortbay.jetty</groupId>
-					<artifactId>jsp-2.1</artifactId>
-				</exclusion>
-				<exclusion>
-					<groupId>org.mortbay.jetty</groupId>
-					<artifactId>jsp-api-2.1</artifactId>
-				</exclusion>
-				<exclusion>
-					<groupId>org.mortbay.jetty</groupId>
-					<artifactId>jetty</artifactId>
-				</exclusion>
-				<exclusion>
-					<groupId>tomcat</groupId>
-					<artifactId>jasper-compiler</artifactId>
-				</exclusion>
-			</exclusions>
+			<artifactId>hbase-client</artifactId>
+			<version>0.96.1.1-cdh5.0.2</version>
 		</dependency>
 		<dependency>
 			<groupId>org.archive.heritrix</groupId>
diff --git a/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java b/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java
index 5bbfec04..c25d78d1 100644
--- a/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java
+++ b/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java
@@ -117,7 +117,7 @@ public class AMQPUrlReceiver implements Lifecycle, ApplicationListener<CrawlStat
                     synchronized (AMQPUrlReceiver.this) {
                         try {
                             Consumer consumer = new UrlConsumer(channel());
-                            channel.queueDeclare(getQueueName(), false, false, true, null);
+                            channel().queueDeclare(getQueueName(), false, false, true, null);
                             channel().queueBind(getQueueName(), getExchange(), getQueueName());
                             channel().basicConsume(getQueueName(), false, consumer);
                             isRunning = true;
diff --git a/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBase.java b/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBase.java
index e4ebd20c..2491771d 100644
--- a/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBase.java
+++ b/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBase.java
@@ -25,8 +25,6 @@ import java.util.logging.Logger;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.MasterNotRunningException;
-import org.apache.hadoop.hbase.ZooKeeperConnectionException;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.springframework.context.Lifecycle;
@@ -78,7 +76,7 @@ public class HBase implements Lifecycle {
 
     protected transient HBaseAdmin admin;
 
-    public synchronized HBaseAdmin admin() throws MasterNotRunningException, ZooKeeperConnectionException {
+    public synchronized HBaseAdmin admin() throws IOException {
         if (admin == null) {
             admin = new HBaseAdmin(configuration());
         }
@@ -99,7 +97,8 @@ public class HBase implements Lifecycle {
             admin = null;
         }
         if (conf != null) {
-            HConnectionManager.deleteConnection(conf, true);
+            // HConnectionManager.deleteConnection(conf); // XXX?
+            conf = null;
         }
     }
 
diff --git a/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBaseContentDigestHistory.java b/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBaseContentDigestHistory.java
index 41197418..6ec29d36 100644
--- a/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBaseContentDigestHistory.java
+++ b/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBaseContentDigestHistory.java
@@ -24,14 +24,15 @@ import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
 import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.thirdparty.guava.common.collect.BiMap;
-import org.apache.hadoop.thirdparty.guava.common.collect.HashBiMap;
 import org.archive.modules.CrawlURI;
 import org.archive.modules.recrawl.AbstractContentDigestHistory;
 import org.json.JSONException;
 import org.json.JSONObject;
 import org.springframework.context.Lifecycle;
 
+import com.google.common.collect.BiMap;
+import com.google.common.collect.HashBiMap;
+
 /**
  * HBase content digest history store. Must be a toplevel bean in
  * crawler-beans.cxml in order to receive {@link Lifecycle} events.
diff --git a/contrib/src/main/java/org/archive/modules/recrawl/hbase/SingleHBaseTable.java b/contrib/src/main/java/org/archive/modules/recrawl/hbase/SingleHBaseTable.java
index f14b98fd..d1eade06 100644
--- a/contrib/src/main/java/org/archive/modules/recrawl/hbase/SingleHBaseTable.java
+++ b/contrib/src/main/java/org/archive/modules/recrawl/hbase/SingleHBaseTable.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.NotServingRegionException;
 import org.apache.hadoop.hbase.TableNotFoundException;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
@@ -158,11 +157,6 @@ public class SingleHBaseTable extends HBaseTableBean {
         } catch (IOException ex) {
             LOG.warn("error closing " + htable + " - some commits may have been lost");
         }
-        // necessary because HTable.close() does neither release HConnection 
-        // resources nor unregister failed HConnection. new HTable will get
-        // the same failed HConnection again without this. Apparently CDH3u5 has
-        // fixed this issue.
-        HConnectionManager.deleteConnection(htable.getConfiguration(), true);
         if (byError) {
             tableError = System.currentTimeMillis();
         }

From 28547989643a8bed0e25294d744a099d47b6e14c Mon Sep 17 00:00:00 2001
From: Noah Levitt <nlevitt@archive.org>
Date: Mon, 7 Jul 2014 19:05:39 -0700
Subject: [PATCH 2/3] new parameter keySuffix-- If not null, keySuffix is
 appended to the lookup key when loading and storing digest history. Thus the
 key looks like {digest}{keySuffix}, e.g. 
 "sha1:22SFHXERHNFOEY6WK7YOUN4PFIPZSB4D-1193". The purpose is to support
 multiple namespaces in a single hbase table, to avoid proliferation of small
 tables. The reason we use a suffix instead of a prefix is to leave open the
 possibility of deduplication across these different namespaces at some point
 in the future.

---
 .../hbase/HBaseContentDigestHistory.java      | 29 +++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBaseContentDigestHistory.java b/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBaseContentDigestHistory.java
index 6ec29d36..e0890ce7 100644
--- a/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBaseContentDigestHistory.java
+++ b/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBaseContentDigestHistory.java
@@ -92,6 +92,35 @@ public class HBaseContentDigestHistory extends AbstractContentDigestHistory impl
         this.maxTries = maxTries;
     }
 
+    protected String keySuffix = null;
+    public String getKeySuffix() {
+        return keySuffix;
+    }
+
+    /**
+     * If not null, keySuffix is appended to the lookup key when loading and
+     * storing digest history. Thus the key looks like {digest}{keySuffix}, e.g.
+     * "sha1:22SFHXERHNFOEY6WK7YOUN4PFIPZSB4D-1193". The purpose is to support
+     * multiple namespaces in a single hbase table, to avoid proliferation of
+     * small tables. The reason we use a suffix instead of a prefix is to leave
+     * open the possibility of deduplication across these different namespaces
+     * at some point in the future.
+     *
+     * @param keySuffix
+     */
+    public void setKeySuffix(String keySuffix) {
+        this.keySuffix = keySuffix;
+    }
+
+    @Override
+    protected String persistKeyFor(CrawlURI curi) {
+        if (keySuffix != null) {
+            return super.persistKeyFor(curi) + keySuffix;
+        } else {
+            return super.persistKeyFor(curi);
+        }
+    }
+
     protected synchronized void addColumnFamily() {
         try {
             HTableDescriptor oldDesc = table.getHtableDescriptor();

From 6e9a0a4d0552b5dc21e9ccc231830de47127e898 Mon Sep 17 00:00:00 2001
From: Noah Levitt <nlevitt@archive.org>
Date: Mon, 7 Jul 2014 19:21:07 -0700
Subject: [PATCH 3/3] only write WARC-Refers-To header when value is not null

---
 .../org/archive/modules/writer/WARCWriterProcessor.java     | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java
index c60c96b9..0c535f87 100644
--- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java
+++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java
@@ -714,8 +714,10 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit
          * with the preceding record which helped determine the present record
          * content."
          */
-        headers.addLabelValue(HEADER_KEY_REFERS_TO, 
-                "<" + curi.getContentDigestHistory().get(A_WARC_RECORD_ID) + ">");
+        if (curi.getContentDigestHistory().get(A_WARC_RECORD_ID) != null) {
+            headers.addLabelValue(HEADER_KEY_REFERS_TO,
+                    "<" + curi.getContentDigestHistory().get(A_WARC_RECORD_ID) + ">");
+        }
         headers.addLabelValue(HEADER_KEY_REFERS_TO_TARGET_URI, 
                 curi.getContentDigestHistory().get(A_ORIGINAL_URL).toString());
         headers.addLabelValue(HEADER_KEY_REFERS_TO_DATE,