From d3e96b27700f146ef3c2599f911b3708451d5ab9 Mon Sep 17 00:00:00 2001 From: Eldon Date: Fri, 24 Jan 2014 18:11:02 -0800 Subject: [PATCH 1/4] First pass at a processor to publish crawluris to AMQP channels --- contrib/pom.xml | 5 + .../archive/modules/AMQPPublishProcessor.java | 121 ++++++++++++++++++ 2 files changed, 126 insertions(+) create mode 100644 contrib/src/main/java/org/archive/modules/AMQPPublishProcessor.java diff --git a/contrib/pom.xml b/contrib/pom.xml index f0994523..6e048084 100644 --- a/contrib/pom.xml +++ b/contrib/pom.xml @@ -54,6 +54,11 @@ ${project.version} compile + + com.rabbitmq + amqp-client + 3.2.1 + diff --git a/contrib/src/main/java/org/archive/modules/AMQPPublishProcessor.java b/contrib/src/main/java/org/archive/modules/AMQPPublishProcessor.java new file mode 100644 index 00000000..38ec3d25 --- /dev/null +++ b/contrib/src/main/java/org/archive/modules/AMQPPublishProcessor.java @@ -0,0 +1,121 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.modules; + +import java.io.IOException; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.json.JSONObject; +import org.springframework.beans.factory.annotation.Required; + +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Connection; +import com.rabbitmq.client.ConnectionFactory; + +/** + + * @author eldondev + * @version $Date$, $Revision$ + */ +public class AMQPPublishProcessor extends Processor { + + @SuppressWarnings("unused") + private static final long serialVersionUID = 1L; + + private static final Logger logger = + Logger.getLogger(AMQPPublishProcessor.class.getName()); + + + protected String amqpUri = null; + protected Connection connection = null; + + public String getAmqpUri() { + return this.amqpUri; + } + + @Required + public void setAmqpUri(String uri) { + this.amqpUri = uri; + } + + transient protected ThreadLocal threadChannel = + new ThreadLocal(); + + /** + * Constructor. + */ + public AMQPPublishProcessor() { + super(); + } + + protected boolean shouldProcess(CrawlURI curi) { + return true; + } + + protected void innerProcess(CrawlURI uri) throws InterruptedException { + try { + Channel channel = getChannel(); + if(channel != null) { + JSONObject message = new JSONObject(); + message.put("url", uri.canonicalString); + channel.basicPublish("umbra", "url", null, message.toString().getBytes()); + } + } catch (Exception e) { + logger.log(Level.SEVERE, "Attempting to send URI to umbra failed!", e); + } + }; + + protected synchronized Channel getChannel() { + if (threadChannel.get() == null) { + if(connection == null) { + connect(); + } + try { + if(connection != null) { + threadChannel.set(connection.createChannel()); + } + } catch (IOException e) { + logger.log(Level.SEVERE, "Attempting to create channel for AMQP connection to umbra failed!", e); + } + } + return threadChannel.get(); + } + + private synchronized void connect() { + ConnectionFactory factory = new ConnectionFactory(); + try { + factory.setUri(amqpUri); + connection = factory.newConnection(); + } catch (Exception e) { + logger.log(Level.SEVERE, "Attempting to connect to AMQP for umbra failed!", e); + } + } + + synchronized public void stop() { + try { + if(connection != null && connection.isOpen()) { + connection.close(); + } + } catch (IOException e) { + logger.log(Level.SEVERE, "Attempting to close AMQP connection failed!", e); + } + } +} From d837536dc67388b531475d4baf56d8c7da3d74e9 Mon Sep 17 00:00:00 2001 From: Eldon Date: Fri, 24 Jan 2014 18:28:38 -0800 Subject: [PATCH 2/4] Use toString method instead --- .../src/main/java/org/archive/modules/AMQPPublishProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/src/main/java/org/archive/modules/AMQPPublishProcessor.java b/contrib/src/main/java/org/archive/modules/AMQPPublishProcessor.java index 38ec3d25..19823c45 100644 --- a/contrib/src/main/java/org/archive/modules/AMQPPublishProcessor.java +++ b/contrib/src/main/java/org/archive/modules/AMQPPublishProcessor.java @@ -75,7 +75,7 @@ public class AMQPPublishProcessor extends Processor { Channel channel = getChannel(); if(channel != null) { JSONObject message = new JSONObject(); - message.put("url", uri.canonicalString); + message.put("url", uri.toString()); channel.basicPublish("umbra", "url", null, message.toString().getBytes()); } } catch (Exception e) { From ab3aadf1999572ed5a69cc337e70dd1c4e9075e0 Mon Sep 17 00:00:00 2001 From: Eldon Date: Fri, 24 Jan 2014 18:47:50 -0800 Subject: [PATCH 3/4] Pull out most umbra refs, only do http/https --- .../archive/modules/AMQPPublishProcessor.java | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/contrib/src/main/java/org/archive/modules/AMQPPublishProcessor.java b/contrib/src/main/java/org/archive/modules/AMQPPublishProcessor.java index 19823c45..e35ac500 100644 --- a/contrib/src/main/java/org/archive/modules/AMQPPublishProcessor.java +++ b/contrib/src/main/java/org/archive/modules/AMQPPublishProcessor.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.util.logging.Level; import java.util.logging.Logger; +import org.archive.modules.fetcher.FetchHTTP; import org.json.JSONObject; import org.springframework.beans.factory.annotation.Required; @@ -59,6 +60,16 @@ public class AMQPPublishProcessor extends Processor { transient protected ThreadLocal threadChannel = new ThreadLocal(); + private String queueName = "umbra"; + + public String getQueueName() { + return queueName; + } + + public void setQueueName(String queueName) { + this.queueName = queueName; + } + /** * Constructor. */ @@ -67,6 +78,10 @@ public class AMQPPublishProcessor extends Processor { } protected boolean shouldProcess(CrawlURI curi) { + if (!(curi.getUURI().getScheme().equals(FetchHTTP.HTTP_SCHEME) || curi.getUURI().getScheme().equals(FetchHTTP.HTTPS_SCHEME))) { + // handles only plain http and https + return false; + } return true; } @@ -76,10 +91,10 @@ public class AMQPPublishProcessor extends Processor { if(channel != null) { JSONObject message = new JSONObject(); message.put("url", uri.toString()); - channel.basicPublish("umbra", "url", null, message.toString().getBytes()); + channel.basicPublish(queueName, "url", null, message.toString().getBytes()); } } catch (Exception e) { - logger.log(Level.SEVERE, "Attempting to send URI to umbra failed!", e); + logger.log(Level.SEVERE, "Attempting to send URI to AMQP server failed!", e); } }; @@ -93,7 +108,7 @@ public class AMQPPublishProcessor extends Processor { threadChannel.set(connection.createChannel()); } } catch (IOException e) { - logger.log(Level.SEVERE, "Attempting to create channel for AMQP connection to umbra failed!", e); + logger.log(Level.SEVERE, "Attempting to create channel for AMQP connection to AMQP server failed!", e); } } return threadChannel.get(); @@ -105,7 +120,7 @@ public class AMQPPublishProcessor extends Processor { factory.setUri(amqpUri); connection = factory.newConnection(); } catch (Exception e) { - logger.log(Level.SEVERE, "Attempting to connect to AMQP for umbra failed!", e); + logger.log(Level.SEVERE, "Attempting to connect to AMQP for AMQP server failed!", e); } } From b33d3faa756dfac94d0a5e1df4feeb358c94e72b Mon Sep 17 00:00:00 2001 From: Eldon Date: Fri, 24 Jan 2014 18:49:04 -0800 Subject: [PATCH 4/4] Better Log messages --- .../main/java/org/archive/modules/AMQPPublishProcessor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/src/main/java/org/archive/modules/AMQPPublishProcessor.java b/contrib/src/main/java/org/archive/modules/AMQPPublishProcessor.java index e35ac500..e9035a8f 100644 --- a/contrib/src/main/java/org/archive/modules/AMQPPublishProcessor.java +++ b/contrib/src/main/java/org/archive/modules/AMQPPublishProcessor.java @@ -108,7 +108,7 @@ public class AMQPPublishProcessor extends Processor { threadChannel.set(connection.createChannel()); } } catch (IOException e) { - logger.log(Level.SEVERE, "Attempting to create channel for AMQP connection to AMQP server failed!", e); + logger.log(Level.SEVERE, "Attempting to create channel for AMQP connection failed!", e); } } return threadChannel.get(); @@ -120,7 +120,7 @@ public class AMQPPublishProcessor extends Processor { factory.setUri(amqpUri); connection = factory.newConnection(); } catch (Exception e) { - logger.log(Level.SEVERE, "Attempting to connect to AMQP for AMQP server failed!", e); + logger.log(Level.SEVERE, "Attempting to connect to AMQP server failed!", e); } }