mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-22 13:47:02 +00:00
fix: make AMQPUrlReceiver resilient to dropped connections
Disable the client's automatic recovery and leave StarterRestarter as the sole owner of the consumer lifecycle, so only one mechanism re-registers a consumer after a failure. Abort dead connections and channels before replacing them, so they cannot go on consuming once the bean has stopped referencing them. Catch everything in handleDelivery, including the json parsing and the ack, so one bad message or one dead channel cannot close the channel out from under every other consumer on it. Hold consumerTag in an AtomicReference and clear it with compareAndSet, so a late shutdown signal cannot clear a newer consumer's tag. A lock would risk deadlocking against stop().
This commit is contained in:
1 parent
be6b49fa6c
commit
d5ec767bff
2 files changed
+270
-32
No files matched your search
@@ -27,6 +27,7 @@ import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import java.util.logging.Level;
|
||||
@@ -56,6 +57,7 @@ import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.context.Lifecycle;
|
||||
|
||||
import com.rabbitmq.client.AMQP.BasicProperties;
|
||||
import com.rabbitmq.client.AlreadyClosedException;
|
||||
import com.rabbitmq.client.Channel;
|
||||
import com.rabbitmq.client.Connection;
|
||||
import com.rabbitmq.client.ConnectionFactory;
|
||||
@@ -164,7 +166,11 @@ public class AMQPUrlReceiver
|
||||
private transient Lock lock = new ReentrantLock(true);
|
||||
|
||||
private transient boolean pauseConsumer = false;
|
||||
private transient String consumerTag = null;
|
||||
/**
|
||||
* Written by StarterRestarter under {@link #lock}, cleared by
|
||||
* handleShutdownSignal on an AMQP client thread that must not take it.
|
||||
*/
|
||||
private transient AtomicReference<String> consumerTag = new AtomicReference<String>();
|
||||
|
||||
private class StarterRestarter extends Thread {
|
||||
|
||||
@@ -177,9 +183,10 @@ public class AMQPUrlReceiver
|
||||
while (!Thread.interrupted()) {
|
||||
try {
|
||||
lock.lockInterruptibly();
|
||||
logger.finest("Checking consumerTag=" + consumerTag + " and pauseConsumer=" + pauseConsumer);
|
||||
String tag = consumerTag.get();
|
||||
logger.finest("Checking consumerTag=" + tag + " and pauseConsumer=" + pauseConsumer);
|
||||
try {
|
||||
if (consumerTag == null && !pauseConsumer) {
|
||||
if (tag == null && !pauseConsumer) {
|
||||
// start up again
|
||||
try {
|
||||
startConsumer();
|
||||
@@ -188,14 +195,12 @@ public class AMQPUrlReceiver
|
||||
}
|
||||
}
|
||||
|
||||
if (consumerTag != null && pauseConsumer) {
|
||||
if (tag != null && pauseConsumer) {
|
||||
try {
|
||||
if (consumerTag != null) {
|
||||
logger.info("Attempting to cancel URLConsumer with consumerTag=" + consumerTag);
|
||||
channel().basicCancel(consumerTag);
|
||||
consumerTag = null;
|
||||
logger.info("Cancelled URLConsumer.");
|
||||
}
|
||||
logger.info("Attempting to cancel URLConsumer with consumerTag=" + tag);
|
||||
channel().basicCancel(tag);
|
||||
consumerTag.compareAndSet(tag, null);
|
||||
logger.info("Cancelled URLConsumer.");
|
||||
} catch (IOException | TimeoutException e) {
|
||||
logger.log(Level.SEVERE, "problem cancelling AMQP consumer (will try again after 10 seconds)", e);
|
||||
}
|
||||
@@ -220,8 +225,9 @@ public class AMQPUrlReceiver
|
||||
channel().queueBind(getQueueName(), getExchange(), getQueueName());
|
||||
if (prefetchCount != null)
|
||||
channel().basicQos(prefetchCount);
|
||||
consumerTag = channel().basicConsume(getQueueName(), false, consumer);
|
||||
logger.info("started AMQP consumer uri=" + getAmqpUri() + " exchange=" + getExchange() + " queueName=" + getQueueName() + " consumerTag=" + consumerTag);
|
||||
String tag = channel().basicConsume(getQueueName(), false, consumer);
|
||||
consumerTag.set(tag);
|
||||
logger.info("started AMQP consumer uri=" + getAmqpUri() + " exchange=" + getExchange() + " queueName=" + getQueueName() + " consumerTag=" + tag);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -286,6 +292,9 @@ public class AMQPUrlReceiver
|
||||
try {
|
||||
if (connection != null && !connection.isOpen()) {
|
||||
logger.warning("connection is closed, creating a new one");
|
||||
// Abort, don't just drop the reference: an unaborted connection
|
||||
// stays live and keeps consuming, invisible to this bean.
|
||||
connection.abort();
|
||||
connection = null;
|
||||
}
|
||||
|
||||
@@ -296,6 +305,9 @@ public class AMQPUrlReceiver
|
||||
} catch (Exception e) {
|
||||
throw new IOException("problem with AMQP uri " + getAmqpUri(), e);
|
||||
}
|
||||
// StarterRestarter owns reconnection. The client's own recovery
|
||||
// would race it and re-register consumers behind its back.
|
||||
factory.setAutomaticRecoveryEnabled(false);
|
||||
connection = factory.newConnection();
|
||||
}
|
||||
|
||||
@@ -310,6 +322,11 @@ public class AMQPUrlReceiver
|
||||
try {
|
||||
if (channel != null && !channel.isOpen()) {
|
||||
logger.warning("channel is not open, creating a new one");
|
||||
try {
|
||||
channel.abort();
|
||||
} catch (IOException e) {
|
||||
logger.log(Level.FINE, "ignoring error aborting dead channel", e);
|
||||
}
|
||||
channel = null;
|
||||
}
|
||||
|
||||
@@ -346,34 +363,40 @@ public class AMQPUrlReceiver
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
throw new RuntimeException(e); // can't happen
|
||||
}
|
||||
JSONObject jo = new JSONObject(decodedBody);
|
||||
try {
|
||||
JSONObject jo = new JSONObject(decodedBody);
|
||||
|
||||
if ("GET".equals(jo.getString("method"))) {
|
||||
try {
|
||||
if ("GET".equals(jo.optString("method"))) {
|
||||
CrawlURI curi = makeCrawlUri(jo);
|
||||
KeyedProperties.clearAllOverrideContexts();
|
||||
candidates.runCandidateChain(curi, null);
|
||||
appCtx.publishEvent(new AMQPUrlReceivedEvent(AMQPUrlReceiver.this, curi));
|
||||
} catch (URIException e) {
|
||||
logger.log(Level.WARNING,
|
||||
"problem creating CrawlURI from json received via AMQP "
|
||||
+ decodedBody, e);
|
||||
} catch (JSONException e) {
|
||||
logger.log(Level.SEVERE,
|
||||
"problem creating CrawlURI from json received via AMQP "
|
||||
+ decodedBody, e);
|
||||
} catch (Exception e) {
|
||||
logger.log(Level.SEVERE,
|
||||
"Unanticipated problem creating CrawlURI from json received via AMQP "
|
||||
+ decodedBody, e);
|
||||
} else {
|
||||
logger.info("ignoring url with method other than GET - "
|
||||
+ decodedBody);
|
||||
}
|
||||
} else {
|
||||
logger.info("ignoring url with method other than GET - "
|
||||
+ decodedBody);
|
||||
} catch (URIException e) {
|
||||
logger.log(Level.WARNING,
|
||||
"problem creating CrawlURI from json received via AMQP "
|
||||
+ decodedBody, e);
|
||||
} catch (JSONException e) {
|
||||
logger.log(Level.SEVERE,
|
||||
"problem creating CrawlURI from json received via AMQP "
|
||||
+ decodedBody, e);
|
||||
} catch (Exception e) {
|
||||
logger.log(Level.SEVERE,
|
||||
"Unanticipated problem creating CrawlURI from json received via AMQP "
|
||||
+ decodedBody, e);
|
||||
}
|
||||
|
||||
logger.finest("Now ACKing: " + decodedBody);
|
||||
this.getChannel().basicAck(envelope.getDeliveryTag(), false);
|
||||
try {
|
||||
this.getChannel().basicAck(envelope.getDeliveryTag(), false);
|
||||
} catch (AlreadyClosedException | IOException e) {
|
||||
// The channel died under us; the broker will redeliver.
|
||||
logger.log(Level.FINE, "could not ACK on a closed channel, "
|
||||
+ "delivery will be redelivered: " + decodedBody, e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -384,7 +407,8 @@ public class AMQPUrlReceiver
|
||||
} else {
|
||||
logger.info("amqp channel/connection shut down consumerTag=" + consumerTag);
|
||||
}
|
||||
AMQPUrlReceiver.this.consumerTag = null;
|
||||
// Only if it is still ours: a newer consumer may already be installed.
|
||||
AMQPUrlReceiver.this.consumerTag.compareAndSet(consumerTag, null);
|
||||
}
|
||||
|
||||
// {
|
||||
|
||||
@@ -24,9 +24,17 @@ import static org.archive.modules.CoreAttributeConstants.A_SOURCE_TAG;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertSame;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.lang.reflect.InvocationHandler;
|
||||
import java.lang.reflect.Method;
|
||||
import java.lang.reflect.Proxy;
|
||||
import java.util.HashSet;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import org.archive.modules.CrawlURI;
|
||||
import org.archive.modules.extractor.Hop;
|
||||
@@ -34,6 +42,12 @@ import org.archive.modules.extractor.LinkContext;
|
||||
import org.json.JSONObject;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import com.rabbitmq.client.AlreadyClosedException;
|
||||
import com.rabbitmq.client.Channel;
|
||||
import com.rabbitmq.client.Connection;
|
||||
import com.rabbitmq.client.Envelope;
|
||||
import com.rabbitmq.client.ShutdownSignalException;
|
||||
|
||||
/**
|
||||
* Tests {@link AMQPUrlReceiver.UrlConsumer#populateHeritableMetadata}, which
|
||||
* copies the {@code heritableData} of an AMQP message into a CrawlURI.
|
||||
@@ -181,4 +195,204 @@ public class AMQPUrlReceiverTest {
|
||||
assertEquals("someValue", curi.getData().get("someOtherKey"));
|
||||
assertEquals(SEED, curi.getSourceTag());
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// connection lifecycle
|
||||
//
|
||||
// The client closes the whole channel when handleDelivery throws, so an
|
||||
// escaping exception takes down every other consumer and in-flight
|
||||
// delivery on it. These pin that nothing escapes, and that dead objects
|
||||
// are aborted rather than merely dereferenced.
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
/** Default return for proxied methods a test does not care about. */
|
||||
protected static Object defaultValue(Method method) {
|
||||
Class<?> t = method.getReturnType();
|
||||
if (t == boolean.class) return false;
|
||||
if (t == int.class) return 0;
|
||||
if (t == long.class) return 0L;
|
||||
if (t == String.class) return "";
|
||||
return null;
|
||||
}
|
||||
|
||||
protected static Channel channelProxy(InvocationHandler handler) {
|
||||
return (Channel) Proxy.newProxyInstance(Channel.class.getClassLoader(),
|
||||
new Class<?>[] { Channel.class }, handler);
|
||||
}
|
||||
|
||||
protected static Connection connectionProxy(InvocationHandler handler) {
|
||||
return (Connection) Proxy.newProxyInstance(Connection.class.getClassLoader(),
|
||||
new Class<?>[] { Connection.class }, handler);
|
||||
}
|
||||
|
||||
protected static Envelope envelope() {
|
||||
return new Envelope(1L, false, "penumbra_exchange", "test_queue");
|
||||
}
|
||||
|
||||
/** A channel that records acks and fails them as a closed channel would. */
|
||||
protected static Channel closedChannelRecording(AtomicBoolean acked) {
|
||||
return channelProxy((proxy, method, args) -> {
|
||||
if ("basicAck".equals(method.getName())) {
|
||||
acked.set(true);
|
||||
throw new AlreadyClosedException(
|
||||
new ShutdownSignalException(false, false, null, null));
|
||||
}
|
||||
return defaultValue(method);
|
||||
});
|
||||
}
|
||||
|
||||
protected static void deliver(Channel channel, String body) throws Exception {
|
||||
AMQPUrlReceiver receiver = new AMQPUrlReceiver();
|
||||
receiver.new UrlConsumer(channel).handleDelivery(
|
||||
"amq.ctag-test", envelope(), null, body.getBytes("UTF-8"));
|
||||
}
|
||||
|
||||
/**
|
||||
* A failed ack must not escape. The broker redelivers, so swallowing it
|
||||
* costs a duplicate; letting it out costs the channel.
|
||||
*/
|
||||
@Test
|
||||
public void testAckFailureOnClosedChannelDoesNotEscape() throws Exception {
|
||||
AtomicBoolean acked = new AtomicBoolean();
|
||||
|
||||
// Not a GET, so the handler skips processing and goes to the ack.
|
||||
deliver(closedChannelRecording(acked),
|
||||
"{\"url\": \"https://example.org/\", \"method\": \"POST\"}");
|
||||
|
||||
assertTrue(acked.get(), "the ack should have been attempted");
|
||||
}
|
||||
|
||||
/**
|
||||
* Malformed json is a poison message: parsing it used to run outside the
|
||||
* try and kill the channel for every other consumer on it.
|
||||
*/
|
||||
@Test
|
||||
public void testMalformedJsonDoesNotEscape() throws Exception {
|
||||
AtomicBoolean acked = new AtomicBoolean();
|
||||
|
||||
deliver(closedChannelRecording(acked), "this is not json");
|
||||
|
||||
assertTrue(acked.get(), "a poison message should still be acked, not redelivered forever");
|
||||
}
|
||||
|
||||
/** A message with no method at all is ignored, not thrown on. */
|
||||
@Test
|
||||
public void testMissingMethodDoesNotEscape() throws Exception {
|
||||
AtomicBoolean acked = new AtomicBoolean();
|
||||
|
||||
deliver(closedChannelRecording(acked), "{\"url\": \"https://example.org/\"}");
|
||||
|
||||
assertTrue(acked.get());
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
protected static AtomicReference<String> consumerTagOf(AMQPUrlReceiver receiver)
|
||||
throws Exception {
|
||||
Field field = AMQPUrlReceiver.class.getDeclaredField("consumerTag");
|
||||
field.setAccessible(true);
|
||||
return (AtomicReference<String>) field.get(receiver);
|
||||
}
|
||||
|
||||
/**
|
||||
* A late shutdown signal must not clear a tag belonging to a consumer that
|
||||
* has since replaced it, or StarterRestarter starts a second consumer
|
||||
* alongside the live one and they accumulate on every failure.
|
||||
*/
|
||||
@Test
|
||||
public void testShutdownSignalLeavesANewerConsumerTagAlone() throws Exception {
|
||||
AMQPUrlReceiver receiver = new AMQPUrlReceiver();
|
||||
AtomicReference<String> tag = consumerTagOf(receiver);
|
||||
tag.set("amq.ctag-new");
|
||||
|
||||
ShutdownSignalException sig = new ShutdownSignalException(false, true, null, null);
|
||||
receiver.new UrlConsumer(null).handleShutdownSignal("amq.ctag-old", sig);
|
||||
|
||||
assertEquals("amq.ctag-new", tag.get());
|
||||
}
|
||||
|
||||
/** The matching signal does still clear it, so the restarter rebuilds. */
|
||||
@Test
|
||||
public void testShutdownSignalClearsItsOwnConsumerTag() throws Exception {
|
||||
AMQPUrlReceiver receiver = new AMQPUrlReceiver();
|
||||
AtomicReference<String> tag = consumerTagOf(receiver);
|
||||
tag.set("amq.ctag-live");
|
||||
|
||||
ShutdownSignalException sig = new ShutdownSignalException(false, true, null, null);
|
||||
receiver.new UrlConsumer(null).handleShutdownSignal("amq.ctag-live", sig);
|
||||
|
||||
assertNull(tag.get());
|
||||
}
|
||||
|
||||
/**
|
||||
* Dropping the reference alone leaves the old channel live and consuming,
|
||||
* invisible to this bean, since the client would go on recovering it.
|
||||
*/
|
||||
@Test
|
||||
public void testDeadChannelIsAbortedBeforeBeingReplaced() throws Exception {
|
||||
AtomicBoolean aborted = new AtomicBoolean();
|
||||
Channel dead = channelProxy((proxy, method, args) -> {
|
||||
switch (method.getName()) {
|
||||
case "isOpen": return false;
|
||||
case "abort": aborted.set(true); return null;
|
||||
default: return defaultValue(method);
|
||||
}
|
||||
});
|
||||
Channel fresh = channelProxy((proxy, method, args) ->
|
||||
"isOpen".equals(method.getName()) ? true : defaultValue(method));
|
||||
Connection connection = connectionProxy((proxy, method, args) -> {
|
||||
switch (method.getName()) {
|
||||
case "isOpen": return true;
|
||||
case "createChannel": return fresh;
|
||||
default: return defaultValue(method);
|
||||
}
|
||||
});
|
||||
|
||||
AMQPUrlReceiver receiver = new AMQPUrlReceiver();
|
||||
receiver.connection = connection;
|
||||
receiver.channel = dead;
|
||||
|
||||
assertSame(fresh, receiver.channel());
|
||||
assertTrue(aborted.get(), "the dead channel should have been aborted");
|
||||
}
|
||||
|
||||
/** The same, one level up: an unaborted connection keeps its consumers. */
|
||||
@Test
|
||||
public void testDeadConnectionIsAbortedBeforeBeingReplaced() throws Exception {
|
||||
AtomicBoolean aborted = new AtomicBoolean();
|
||||
Connection dead = connectionProxy((proxy, method, args) -> {
|
||||
switch (method.getName()) {
|
||||
case "isOpen": return false;
|
||||
case "abort": aborted.set(true); return null;
|
||||
default: return defaultValue(method);
|
||||
}
|
||||
});
|
||||
|
||||
AMQPUrlReceiver receiver = new AMQPUrlReceiver();
|
||||
receiver.setAmqpUri("amqp://guest:guest@127.0.0.1:1/%2f");
|
||||
receiver.connection = dead;
|
||||
|
||||
// The replacement attempt fails -- nothing is listening -- but the
|
||||
// abort must already have happened by then.
|
||||
assertThrows(Exception.class, () -> receiver.connection());
|
||||
assertTrue(aborted.get(), "the dead connection should have been aborted");
|
||||
}
|
||||
|
||||
/** An open channel is handed back untouched. */
|
||||
@Test
|
||||
public void testOpenChannelIsReused() throws Exception {
|
||||
AtomicBoolean aborted = new AtomicBoolean();
|
||||
Channel open = channelProxy((proxy, method, args) -> {
|
||||
switch (method.getName()) {
|
||||
case "isOpen": return true;
|
||||
case "abort": aborted.set(true); return null;
|
||||
default: return defaultValue(method);
|
||||
}
|
||||
});
|
||||
|
||||
AMQPUrlReceiver receiver = new AMQPUrlReceiver();
|
||||
receiver.channel = open;
|
||||
|
||||
assertSame(open, receiver.channel());
|
||||
assertFalse(aborted.get());
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user