lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-25 07:05:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #81 from nlevitt/hbase-cdh5

Browse Source 
Hbase cdh5
This commit is contained in:
Vinay Goel
2014-07-23 13:34:54 -07:00
parent ba48f96d93 40eae4cbce
commit 1c6746c6c8
5 changed files with 38 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
contrib/pom.xml
+2 -25
View File
@@ -16,31 +16,8 @@
<dependencies>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase</artifactId>
<version>0.90.6-cdh3u5</version>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>servlet-api-2.5</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jsp-2.1</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jsp-api-2.1</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
</exclusion>
<exclusion>
<groupId>tomcat</groupId>
<artifactId>jasper-compiler</artifactId>
</exclusion>
</exclusions>
<artifactId>hbase-client</artifactId>
<version>0.96.1.1-cdh5.0.2</version>
</dependency>
<dependency>
<groupId>org.archive.heritrix</groupId>
contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java
+1 -1
View File
@@ -117,7 +117,7 @@ public class AMQPUrlReceiver implements Lifecycle, ApplicationListener<CrawlStat
synchronized (AMQPUrlReceiver.this) {
try {
Consumer consumer = new UrlConsumer(channel());
channel.queueDeclare(getQueueName(), false, false, true, null);
channel().queueDeclare(getQueueName(), false, false, true, null);
channel().queueBind(getQueueName(), getExchange(), getQueueName());
channel().basicConsume(getQueueName(), false, consumer);
isRunning = true;
contrib/src/main/java/org/archive/modules/recrawl/hbase/HBase.java
+3 -4
View File
@@ -25,8 +25,6 @@ import java.util.logging.Logger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.springframework.context.Lifecycle;
@@ -78,7 +76,7 @@ public class HBase implements Lifecycle {
protected transient HBaseAdmin admin;
public synchronized HBaseAdmin admin() throws MasterNotRunningException, ZooKeeperConnectionException {
public synchronized HBaseAdmin admin() throws IOException {
if (admin == null) {
admin = new HBaseAdmin(configuration());
}
@@ -99,7 +97,8 @@ public class HBase implements Lifecycle {
admin = null;
}
if (conf != null) {
HConnectionManager.deleteConnection(conf, true);
// HConnectionManager.deleteConnection(conf); // XXX?
conf = null;
}
}
contrib/src/main/java/org/archive/modules/recrawl/hbase/HBaseContentDigestHistory.java
+32 -2
View File
@@ -24,14 +24,15 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.thirdparty.guava.common.collect.BiMap;
import org.apache.hadoop.thirdparty.guava.common.collect.HashBiMap;
import org.archive.modules.CrawlURI;
import org.archive.modules.recrawl.AbstractContentDigestHistory;
import org.json.JSONException;
import org.json.JSONObject;
import org.springframework.context.Lifecycle;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
/**
* HBase content digest history store. Must be a toplevel bean in
* crawler-beans.cxml in order to receive {@link Lifecycle} events.
@@ -91,6 +92,35 @@ public class HBaseContentDigestHistory extends AbstractContentDigestHistory impl
this.maxTries = maxTries;
}
protected String keySuffix = null;
public String getKeySuffix() {
return keySuffix;
}
/**
* If not null, keySuffix is appended to the lookup key when loading and
* storing digest history. Thus the key looks like {digest}{keySuffix}, e.g.
* "sha1:22SFHXERHNFOEY6WK7YOUN4PFIPZSB4D-1193". The purpose is to support
* multiple namespaces in a single hbase table, to avoid proliferation of
* small tables. The reason we use a suffix instead of a prefix is to leave
* open the possibility of deduplication across these different namespaces
* at some point in the future.
*
* @param keySuffix
*/
public void setKeySuffix(String keySuffix) {
this.keySuffix = keySuffix;
}
@Override
protected String persistKeyFor(CrawlURI curi) {
if (keySuffix != null) {
return super.persistKeyFor(curi) + keySuffix;
} else {
return super.persistKeyFor(curi);
}
}
protected synchronized void addColumnFamily() {
try {
HTableDescriptor oldDesc = table.getHtableDescriptor();
contrib/src/main/java/org/archive/modules/recrawl/hbase/SingleHBaseTable.java
-6
View File
@@ -32,7 +32,6 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.NotServingRegionException;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
@@ -158,11 +157,6 @@ public class SingleHBaseTable extends HBaseTableBean {
} catch (IOException ex) {
LOG.warn("error closing " + htable + " - some commits may have been lost");
}
// necessary because HTable.close() does neither release HConnection
// resources nor unregister failed HConnection. new HTable will get
// the same failed HConnection again without this. Apparently CDH3u5 has
// fixed this issue.
HConnectionManager.deleteConnection(htable.getConfiguration(), true);
if (byError) {
tableError = System.currentTimeMillis();
}