From d5ec767bffbb2e090635b7d5f49a619efeedf508 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Mon, 5 Oct 2026 17:27:30 -0700 Subject: [PATCH] fix: make AMQPUrlReceiver resilient to dropped connections Disable the client's automatic recovery and leave StarterRestarter as the sole owner of the consumer lifecycle, so only one mechanism re-registers a consumer after a failure. Abort dead connections and channels before replacing them, so they cannot go on consuming once the bean has stopped referencing them. Catch everything in handleDelivery, including the json parsing and the ack, so one bad message or one dead channel cannot close the channel out from under every other consumer on it. Hold consumerTag in an AtomicReference and clear it with compareAndSet, so a late shutdown signal cannot clear a newer consumer's tag. A lock would risk deadlocking against stop(). --- .../crawler/frontier/AMQPUrlReceiver.java | 88 ++++--- .../crawler/frontier/AMQPUrlReceiverTest.java | 214 ++++++++++++++++++ 2 files changed, 270 insertions(+), 32 deletions(-) diff --git a/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java b/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java index b0d326b6..d8c2f997 100644 --- a/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java +++ b/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java @@ -27,6 +27,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.logging.Level; @@ -56,6 +57,7 @@ import org.springframework.context.ApplicationListener; import org.springframework.context.Lifecycle; import com.rabbitmq.client.AMQP.BasicProperties; +import com.rabbitmq.client.AlreadyClosedException; import com.rabbitmq.client.Channel; import com.rabbitmq.client.Connection; import com.rabbitmq.client.ConnectionFactory; @@ -164,7 +166,11 @@ public class AMQPUrlReceiver private transient Lock lock = new ReentrantLock(true); private transient boolean pauseConsumer = false; - private transient String consumerTag = null; + /** + * Written by StarterRestarter under {@link #lock}, cleared by + * handleShutdownSignal on an AMQP client thread that must not take it. + */ + private transient AtomicReference consumerTag = new AtomicReference(); private class StarterRestarter extends Thread { @@ -177,9 +183,10 @@ public class AMQPUrlReceiver while (!Thread.interrupted()) { try { lock.lockInterruptibly(); - logger.finest("Checking consumerTag=" + consumerTag + " and pauseConsumer=" + pauseConsumer); + String tag = consumerTag.get(); + logger.finest("Checking consumerTag=" + tag + " and pauseConsumer=" + pauseConsumer); try { - if (consumerTag == null && !pauseConsumer) { + if (tag == null && !pauseConsumer) { // start up again try { startConsumer(); @@ -188,14 +195,12 @@ public class AMQPUrlReceiver } } - if (consumerTag != null && pauseConsumer) { + if (tag != null && pauseConsumer) { try { - if (consumerTag != null) { - logger.info("Attempting to cancel URLConsumer with consumerTag=" + consumerTag); - channel().basicCancel(consumerTag); - consumerTag = null; - logger.info("Cancelled URLConsumer."); - } + logger.info("Attempting to cancel URLConsumer with consumerTag=" + tag); + channel().basicCancel(tag); + consumerTag.compareAndSet(tag, null); + logger.info("Cancelled URLConsumer."); } catch (IOException | TimeoutException e) { logger.log(Level.SEVERE, "problem cancelling AMQP consumer (will try again after 10 seconds)", e); } @@ -220,8 +225,9 @@ public class AMQPUrlReceiver channel().queueBind(getQueueName(), getExchange(), getQueueName()); if (prefetchCount != null) channel().basicQos(prefetchCount); - consumerTag = channel().basicConsume(getQueueName(), false, consumer); - logger.info("started AMQP consumer uri=" + getAmqpUri() + " exchange=" + getExchange() + " queueName=" + getQueueName() + " consumerTag=" + consumerTag); + String tag = channel().basicConsume(getQueueName(), false, consumer); + consumerTag.set(tag); + logger.info("started AMQP consumer uri=" + getAmqpUri() + " exchange=" + getExchange() + " queueName=" + getQueueName() + " consumerTag=" + tag); } } @@ -286,6 +292,9 @@ public class AMQPUrlReceiver try { if (connection != null && !connection.isOpen()) { logger.warning("connection is closed, creating a new one"); + // Abort, don't just drop the reference: an unaborted connection + // stays live and keeps consuming, invisible to this bean. + connection.abort(); connection = null; } @@ -296,6 +305,9 @@ public class AMQPUrlReceiver } catch (Exception e) { throw new IOException("problem with AMQP uri " + getAmqpUri(), e); } + // StarterRestarter owns reconnection. The client's own recovery + // would race it and re-register consumers behind its back. + factory.setAutomaticRecoveryEnabled(false); connection = factory.newConnection(); } @@ -310,6 +322,11 @@ public class AMQPUrlReceiver try { if (channel != null && !channel.isOpen()) { logger.warning("channel is not open, creating a new one"); + try { + channel.abort(); + } catch (IOException e) { + logger.log(Level.FINE, "ignoring error aborting dead channel", e); + } channel = null; } @@ -346,34 +363,40 @@ public class AMQPUrlReceiver } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); // can't happen } - JSONObject jo = new JSONObject(decodedBody); + try { + JSONObject jo = new JSONObject(decodedBody); - if ("GET".equals(jo.getString("method"))) { - try { + if ("GET".equals(jo.optString("method"))) { CrawlURI curi = makeCrawlUri(jo); KeyedProperties.clearAllOverrideContexts(); candidates.runCandidateChain(curi, null); appCtx.publishEvent(new AMQPUrlReceivedEvent(AMQPUrlReceiver.this, curi)); - } catch (URIException e) { - logger.log(Level.WARNING, - "problem creating CrawlURI from json received via AMQP " - + decodedBody, e); - } catch (JSONException e) { - logger.log(Level.SEVERE, - "problem creating CrawlURI from json received via AMQP " - + decodedBody, e); - } catch (Exception e) { - logger.log(Level.SEVERE, - "Unanticipated problem creating CrawlURI from json received via AMQP " - + decodedBody, e); + } else { + logger.info("ignoring url with method other than GET - " + + decodedBody); } - } else { - logger.info("ignoring url with method other than GET - " - + decodedBody); + } catch (URIException e) { + logger.log(Level.WARNING, + "problem creating CrawlURI from json received via AMQP " + + decodedBody, e); + } catch (JSONException e) { + logger.log(Level.SEVERE, + "problem creating CrawlURI from json received via AMQP " + + decodedBody, e); + } catch (Exception e) { + logger.log(Level.SEVERE, + "Unanticipated problem creating CrawlURI from json received via AMQP " + + decodedBody, e); } logger.finest("Now ACKing: " + decodedBody); - this.getChannel().basicAck(envelope.getDeliveryTag(), false); + try { + this.getChannel().basicAck(envelope.getDeliveryTag(), false); + } catch (AlreadyClosedException | IOException e) { + // The channel died under us; the broker will redeliver. + logger.log(Level.FINE, "could not ACK on a closed channel, " + + "delivery will be redelivered: " + decodedBody, e); + } } @Override @@ -384,7 +407,8 @@ public class AMQPUrlReceiver } else { logger.info("amqp channel/connection shut down consumerTag=" + consumerTag); } - AMQPUrlReceiver.this.consumerTag = null; + // Only if it is still ours: a newer consumer may already be installed. + AMQPUrlReceiver.this.consumerTag.compareAndSet(consumerTag, null); } // { diff --git a/contrib/src/test/java/org/archive/crawler/frontier/AMQPUrlReceiverTest.java b/contrib/src/test/java/org/archive/crawler/frontier/AMQPUrlReceiverTest.java index 2de4e658..35d64f7d 100644 --- a/contrib/src/test/java/org/archive/crawler/frontier/AMQPUrlReceiverTest.java +++ b/contrib/src/test/java/org/archive/crawler/frontier/AMQPUrlReceiverTest.java @@ -24,9 +24,17 @@ import static org.archive.modules.CoreAttributeConstants.A_SOURCE_TAG; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; import java.util.HashSet; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import org.archive.modules.CrawlURI; import org.archive.modules.extractor.Hop; @@ -34,6 +42,12 @@ import org.archive.modules.extractor.LinkContext; import org.json.JSONObject; import org.junit.jupiter.api.Test; +import com.rabbitmq.client.AlreadyClosedException; +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Connection; +import com.rabbitmq.client.Envelope; +import com.rabbitmq.client.ShutdownSignalException; + /** * Tests {@link AMQPUrlReceiver.UrlConsumer#populateHeritableMetadata}, which * copies the {@code heritableData} of an AMQP message into a CrawlURI. @@ -181,4 +195,204 @@ public class AMQPUrlReceiverTest { assertEquals("someValue", curi.getData().get("someOtherKey")); assertEquals(SEED, curi.getSourceTag()); } + + // --------------------------------------------------------------- + // connection lifecycle + // + // The client closes the whole channel when handleDelivery throws, so an + // escaping exception takes down every other consumer and in-flight + // delivery on it. These pin that nothing escapes, and that dead objects + // are aborted rather than merely dereferenced. + // --------------------------------------------------------------- + + /** Default return for proxied methods a test does not care about. */ + protected static Object defaultValue(Method method) { + Class t = method.getReturnType(); + if (t == boolean.class) return false; + if (t == int.class) return 0; + if (t == long.class) return 0L; + if (t == String.class) return ""; + return null; + } + + protected static Channel channelProxy(InvocationHandler handler) { + return (Channel) Proxy.newProxyInstance(Channel.class.getClassLoader(), + new Class[] { Channel.class }, handler); + } + + protected static Connection connectionProxy(InvocationHandler handler) { + return (Connection) Proxy.newProxyInstance(Connection.class.getClassLoader(), + new Class[] { Connection.class }, handler); + } + + protected static Envelope envelope() { + return new Envelope(1L, false, "penumbra_exchange", "test_queue"); + } + + /** A channel that records acks and fails them as a closed channel would. */ + protected static Channel closedChannelRecording(AtomicBoolean acked) { + return channelProxy((proxy, method, args) -> { + if ("basicAck".equals(method.getName())) { + acked.set(true); + throw new AlreadyClosedException( + new ShutdownSignalException(false, false, null, null)); + } + return defaultValue(method); + }); + } + + protected static void deliver(Channel channel, String body) throws Exception { + AMQPUrlReceiver receiver = new AMQPUrlReceiver(); + receiver.new UrlConsumer(channel).handleDelivery( + "amq.ctag-test", envelope(), null, body.getBytes("UTF-8")); + } + + /** + * A failed ack must not escape. The broker redelivers, so swallowing it + * costs a duplicate; letting it out costs the channel. + */ + @Test + public void testAckFailureOnClosedChannelDoesNotEscape() throws Exception { + AtomicBoolean acked = new AtomicBoolean(); + + // Not a GET, so the handler skips processing and goes to the ack. + deliver(closedChannelRecording(acked), + "{\"url\": \"https://example.org/\", \"method\": \"POST\"}"); + + assertTrue(acked.get(), "the ack should have been attempted"); + } + + /** + * Malformed json is a poison message: parsing it used to run outside the + * try and kill the channel for every other consumer on it. + */ + @Test + public void testMalformedJsonDoesNotEscape() throws Exception { + AtomicBoolean acked = new AtomicBoolean(); + + deliver(closedChannelRecording(acked), "this is not json"); + + assertTrue(acked.get(), "a poison message should still be acked, not redelivered forever"); + } + + /** A message with no method at all is ignored, not thrown on. */ + @Test + public void testMissingMethodDoesNotEscape() throws Exception { + AtomicBoolean acked = new AtomicBoolean(); + + deliver(closedChannelRecording(acked), "{\"url\": \"https://example.org/\"}"); + + assertTrue(acked.get()); + } + + @SuppressWarnings("unchecked") + protected static AtomicReference consumerTagOf(AMQPUrlReceiver receiver) + throws Exception { + Field field = AMQPUrlReceiver.class.getDeclaredField("consumerTag"); + field.setAccessible(true); + return (AtomicReference) field.get(receiver); + } + + /** + * A late shutdown signal must not clear a tag belonging to a consumer that + * has since replaced it, or StarterRestarter starts a second consumer + * alongside the live one and they accumulate on every failure. + */ + @Test + public void testShutdownSignalLeavesANewerConsumerTagAlone() throws Exception { + AMQPUrlReceiver receiver = new AMQPUrlReceiver(); + AtomicReference tag = consumerTagOf(receiver); + tag.set("amq.ctag-new"); + + ShutdownSignalException sig = new ShutdownSignalException(false, true, null, null); + receiver.new UrlConsumer(null).handleShutdownSignal("amq.ctag-old", sig); + + assertEquals("amq.ctag-new", tag.get()); + } + + /** The matching signal does still clear it, so the restarter rebuilds. */ + @Test + public void testShutdownSignalClearsItsOwnConsumerTag() throws Exception { + AMQPUrlReceiver receiver = new AMQPUrlReceiver(); + AtomicReference tag = consumerTagOf(receiver); + tag.set("amq.ctag-live"); + + ShutdownSignalException sig = new ShutdownSignalException(false, true, null, null); + receiver.new UrlConsumer(null).handleShutdownSignal("amq.ctag-live", sig); + + assertNull(tag.get()); + } + + /** + * Dropping the reference alone leaves the old channel live and consuming, + * invisible to this bean, since the client would go on recovering it. + */ + @Test + public void testDeadChannelIsAbortedBeforeBeingReplaced() throws Exception { + AtomicBoolean aborted = new AtomicBoolean(); + Channel dead = channelProxy((proxy, method, args) -> { + switch (method.getName()) { + case "isOpen": return false; + case "abort": aborted.set(true); return null; + default: return defaultValue(method); + } + }); + Channel fresh = channelProxy((proxy, method, args) -> + "isOpen".equals(method.getName()) ? true : defaultValue(method)); + Connection connection = connectionProxy((proxy, method, args) -> { + switch (method.getName()) { + case "isOpen": return true; + case "createChannel": return fresh; + default: return defaultValue(method); + } + }); + + AMQPUrlReceiver receiver = new AMQPUrlReceiver(); + receiver.connection = connection; + receiver.channel = dead; + + assertSame(fresh, receiver.channel()); + assertTrue(aborted.get(), "the dead channel should have been aborted"); + } + + /** The same, one level up: an unaborted connection keeps its consumers. */ + @Test + public void testDeadConnectionIsAbortedBeforeBeingReplaced() throws Exception { + AtomicBoolean aborted = new AtomicBoolean(); + Connection dead = connectionProxy((proxy, method, args) -> { + switch (method.getName()) { + case "isOpen": return false; + case "abort": aborted.set(true); return null; + default: return defaultValue(method); + } + }); + + AMQPUrlReceiver receiver = new AMQPUrlReceiver(); + receiver.setAmqpUri("amqp://guest:guest@127.0.0.1:1/%2f"); + receiver.connection = dead; + + // The replacement attempt fails -- nothing is listening -- but the + // abort must already have happened by then. + assertThrows(Exception.class, () -> receiver.connection()); + assertTrue(aborted.get(), "the dead connection should have been aborted"); + } + + /** An open channel is handed back untouched. */ + @Test + public void testOpenChannelIsReused() throws Exception { + AtomicBoolean aborted = new AtomicBoolean(); + Channel open = channelProxy((proxy, method, args) -> { + switch (method.getName()) { + case "isOpen": return true; + case "abort": aborted.set(true); return null; + default: return defaultValue(method); + } + }); + + AMQPUrlReceiver receiver = new AMQPUrlReceiver(); + receiver.channel = open; + + assertSame(open, receiver.channel()); + assertFalse(aborted.get()); + } }