lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-23 06:05:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

upgrade hbase to cdh5

Browse Source
This commit is contained in:
Noah Levitt
2014-06-16 18:33:25 -07:00
parent 8c7b0fed92
commit 3e35c967ab
5 changed files with 9 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
contrib/pom.xml
+2 -25
View File
@@ -16,31 +16,8 @@
<dependencies>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase</artifactId>
<version>0.90.6-cdh3u5</version>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>servlet-api-2.5</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jsp-2.1</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jsp-api-2.1</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
</exclusion>
<exclusion>
<groupId>tomcat</groupId>
<artifactId>jasper-compiler</artifactId>
</exclusion>
</exclusions>
<artifactId>hbase-client</artifactId>
<version>0.96.1.1-cdh5.0.2</version>
</dependency>
<dependency>
<groupId>org.archive.heritrix</groupId>
contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java
+1 -1
View File
@@ -117,7 +117,7 @@ public class AMQPUrlReceiver implements Lifecycle, ApplicationListener<CrawlStat
synchronized (AMQPUrlReceiver.this) {
try {
Consumer consumer = new UrlConsumer(channel());
channel.queueDeclare(getQueueName(), false, false, true, null);
channel().queueDeclare(getQueueName(), false, false, true, null);
channel().queueBind(getQueueName(), getExchange(), getQueueName());
channel().basicConsume(getQueueName(), false, consumer);
isRunning = true;
contrib/src/main/java/org/archive/modules/recrawl/hbase/HBase.java
+3 -4
View File
@@ -25,8 +25,6 @@ import java.util.logging.Logger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.springframework.context.Lifecycle;
@@ -78,7 +76,7 @@ public class HBase implements Lifecycle {
protected transient HBaseAdmin admin;
public synchronized HBaseAdmin admin() throws MasterNotRunningException, ZooKeeperConnectionException {
public synchronized HBaseAdmin admin() throws IOException {
if (admin == null) {
admin = new HBaseAdmin(configuration());
}
@@ -99,7 +97,8 @@ public class HBase implements Lifecycle {
admin = null;
}
if (conf != null) {
HConnectionManager.deleteConnection(conf, true);
// HConnectionManager.deleteConnection(conf); // XXX?
conf = null;
}
}
contrib/src/main/java/org/archive/modules/recrawl/hbase/HBaseContentDigestHistory.java
+3 -2
View File
@@ -24,14 +24,15 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.thirdparty.guava.common.collect.BiMap;
import org.apache.hadoop.thirdparty.guava.common.collect.HashBiMap;
import org.archive.modules.CrawlURI;
import org.archive.modules.recrawl.AbstractContentDigestHistory;
import org.json.JSONException;
import org.json.JSONObject;
import org.springframework.context.Lifecycle;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
/**
* HBase content digest history store. Must be a toplevel bean in
* crawler-beans.cxml in order to receive {@link Lifecycle} events.
contrib/src/main/java/org/archive/modules/recrawl/hbase/SingleHBaseTable.java
-6
View File
@@ -32,7 +32,6 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.NotServingRegionException;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
@@ -158,11 +157,6 @@ public class SingleHBaseTable extends HBaseTableBean {
} catch (IOException ex) {
LOG.warn("error closing " + htable + " - some commits may have been lost");
}
// necessary because HTable.close() does neither release HConnection
// resources nor unregister failed HConnection. new HTable will get
// the same failed HConnection again without this. Apparently CDH3u5 has
// fixed this issue.
HConnectionManager.deleteConnection(htable.getConfiguration(), true);
if (byError) {
tableError = System.currentTimeMillis();
}