diff --git a/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java b/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java index b0d326b6..d8c2f997 100644 --- a/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java +++ b/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java @@ -27,6 +27,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.logging.Level; @@ -56,6 +57,7 @@ import org.springframework.context.ApplicationListener; import org.springframework.context.Lifecycle; import com.rabbitmq.client.AMQP.BasicProperties; +import com.rabbitmq.client.AlreadyClosedException; import com.rabbitmq.client.Channel; import com.rabbitmq.client.Connection; import com.rabbitmq.client.ConnectionFactory; @@ -164,7 +166,11 @@ public class AMQPUrlReceiver private transient Lock lock = new ReentrantLock(true); private transient boolean pauseConsumer = false; - private transient String consumerTag = null; + /** + * Written by StarterRestarter under {@link #lock}, cleared by + * handleShutdownSignal on an AMQP client thread that must not take it. + */ + private transient AtomicReference consumerTag = new AtomicReference(); private class StarterRestarter extends Thread { @@ -177,9 +183,10 @@ public class AMQPUrlReceiver while (!Thread.interrupted()) { try { lock.lockInterruptibly(); - logger.finest("Checking consumerTag=" + consumerTag + " and pauseConsumer=" + pauseConsumer); + String tag = consumerTag.get(); + logger.finest("Checking consumerTag=" + tag + " and pauseConsumer=" + pauseConsumer); try { - if (consumerTag == null && !pauseConsumer) { + if (tag == null && !pauseConsumer) { // start up again try { startConsumer(); @@ -188,14 +195,12 @@ public class AMQPUrlReceiver } } - if (consumerTag != null && pauseConsumer) { + if (tag != null && pauseConsumer) { try { - if (consumerTag != null) { - logger.info("Attempting to cancel URLConsumer with consumerTag=" + consumerTag); - channel().basicCancel(consumerTag); - consumerTag = null; - logger.info("Cancelled URLConsumer."); - } + logger.info("Attempting to cancel URLConsumer with consumerTag=" + tag); + channel().basicCancel(tag); + consumerTag.compareAndSet(tag, null); + logger.info("Cancelled URLConsumer."); } catch (IOException | TimeoutException e) { logger.log(Level.SEVERE, "problem cancelling AMQP consumer (will try again after 10 seconds)", e); } @@ -220,8 +225,9 @@ public class AMQPUrlReceiver channel().queueBind(getQueueName(), getExchange(), getQueueName()); if (prefetchCount != null) channel().basicQos(prefetchCount); - consumerTag = channel().basicConsume(getQueueName(), false, consumer); - logger.info("started AMQP consumer uri=" + getAmqpUri() + " exchange=" + getExchange() + " queueName=" + getQueueName() + " consumerTag=" + consumerTag); + String tag = channel().basicConsume(getQueueName(), false, consumer); + consumerTag.set(tag); + logger.info("started AMQP consumer uri=" + getAmqpUri() + " exchange=" + getExchange() + " queueName=" + getQueueName() + " consumerTag=" + tag); } } @@ -286,6 +292,9 @@ public class AMQPUrlReceiver try { if (connection != null && !connection.isOpen()) { logger.warning("connection is closed, creating a new one"); + // Abort, don't just drop the reference: an unaborted connection + // stays live and keeps consuming, invisible to this bean. + connection.abort(); connection = null; } @@ -296,6 +305,9 @@ public class AMQPUrlReceiver } catch (Exception e) { throw new IOException("problem with AMQP uri " + getAmqpUri(), e); } + // StarterRestarter owns reconnection. The client's own recovery + // would race it and re-register consumers behind its back. + factory.setAutomaticRecoveryEnabled(false); connection = factory.newConnection(); } @@ -310,6 +322,11 @@ public class AMQPUrlReceiver try { if (channel != null && !channel.isOpen()) { logger.warning("channel is not open, creating a new one"); + try { + channel.abort(); + } catch (IOException e) { + logger.log(Level.FINE, "ignoring error aborting dead channel", e); + } channel = null; } @@ -346,34 +363,40 @@ public class AMQPUrlReceiver } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); // can't happen } - JSONObject jo = new JSONObject(decodedBody); + try { + JSONObject jo = new JSONObject(decodedBody); - if ("GET".equals(jo.getString("method"))) { - try { + if ("GET".equals(jo.optString("method"))) { CrawlURI curi = makeCrawlUri(jo); KeyedProperties.clearAllOverrideContexts(); candidates.runCandidateChain(curi, null); appCtx.publishEvent(new AMQPUrlReceivedEvent(AMQPUrlReceiver.this, curi)); - } catch (URIException e) { - logger.log(Level.WARNING, - "problem creating CrawlURI from json received via AMQP " - + decodedBody, e); - } catch (JSONException e) { - logger.log(Level.SEVERE, - "problem creating CrawlURI from json received via AMQP " - + decodedBody, e); - } catch (Exception e) { - logger.log(Level.SEVERE, - "Unanticipated problem creating CrawlURI from json received via AMQP " - + decodedBody, e); + } else { + logger.info("ignoring url with method other than GET - " + + decodedBody); } - } else { - logger.info("ignoring url with method other than GET - " - + decodedBody); + } catch (URIException e) { + logger.log(Level.WARNING, + "problem creating CrawlURI from json received via AMQP " + + decodedBody, e); + } catch (JSONException e) { + logger.log(Level.SEVERE, + "problem creating CrawlURI from json received via AMQP " + + decodedBody, e); + } catch (Exception e) { + logger.log(Level.SEVERE, + "Unanticipated problem creating CrawlURI from json received via AMQP " + + decodedBody, e); } logger.finest("Now ACKing: " + decodedBody); - this.getChannel().basicAck(envelope.getDeliveryTag(), false); + try { + this.getChannel().basicAck(envelope.getDeliveryTag(), false); + } catch (AlreadyClosedException | IOException e) { + // The channel died under us; the broker will redeliver. + logger.log(Level.FINE, "could not ACK on a closed channel, " + + "delivery will be redelivered: " + decodedBody, e); + } } @Override @@ -384,7 +407,8 @@ public class AMQPUrlReceiver } else { logger.info("amqp channel/connection shut down consumerTag=" + consumerTag); } - AMQPUrlReceiver.this.consumerTag = null; + // Only if it is still ours: a newer consumer may already be installed. + AMQPUrlReceiver.this.consumerTag.compareAndSet(consumerTag, null); } // { diff --git a/contrib/src/test/java/org/archive/crawler/frontier/AMQPUrlReceiverTest.java b/contrib/src/test/java/org/archive/crawler/frontier/AMQPUrlReceiverTest.java index 2de4e658..35d64f7d 100644 --- a/contrib/src/test/java/org/archive/crawler/frontier/AMQPUrlReceiverTest.java +++ b/contrib/src/test/java/org/archive/crawler/frontier/AMQPUrlReceiverTest.java @@ -24,9 +24,17 @@ import static org.archive.modules.CoreAttributeConstants.A_SOURCE_TAG; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; import java.util.HashSet; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import org.archive.modules.CrawlURI; import org.archive.modules.extractor.Hop; @@ -34,6 +42,12 @@ import org.archive.modules.extractor.LinkContext; import org.json.JSONObject; import org.junit.jupiter.api.Test; +import com.rabbitmq.client.AlreadyClosedException; +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Connection; +import com.rabbitmq.client.Envelope; +import com.rabbitmq.client.ShutdownSignalException; + /** * Tests {@link AMQPUrlReceiver.UrlConsumer#populateHeritableMetadata}, which * copies the {@code heritableData} of an AMQP message into a CrawlURI. @@ -181,4 +195,204 @@ public class AMQPUrlReceiverTest { assertEquals("someValue", curi.getData().get("someOtherKey")); assertEquals(SEED, curi.getSourceTag()); } + + // --------------------------------------------------------------- + // connection lifecycle + // + // The client closes the whole channel when handleDelivery throws, so an + // escaping exception takes down every other consumer and in-flight + // delivery on it. These pin that nothing escapes, and that dead objects + // are aborted rather than merely dereferenced. + // --------------------------------------------------------------- + + /** Default return for proxied methods a test does not care about. */ + protected static Object defaultValue(Method method) { + Class t = method.getReturnType(); + if (t == boolean.class) return false; + if (t == int.class) return 0; + if (t == long.class) return 0L; + if (t == String.class) return ""; + return null; + } + + protected static Channel channelProxy(InvocationHandler handler) { + return (Channel) Proxy.newProxyInstance(Channel.class.getClassLoader(), + new Class[] { Channel.class }, handler); + } + + protected static Connection connectionProxy(InvocationHandler handler) { + return (Connection) Proxy.newProxyInstance(Connection.class.getClassLoader(), + new Class[] { Connection.class }, handler); + } + + protected static Envelope envelope() { + return new Envelope(1L, false, "penumbra_exchange", "test_queue"); + } + + /** A channel that records acks and fails them as a closed channel would. */ + protected static Channel closedChannelRecording(AtomicBoolean acked) { + return channelProxy((proxy, method, args) -> { + if ("basicAck".equals(method.getName())) { + acked.set(true); + throw new AlreadyClosedException( + new ShutdownSignalException(false, false, null, null)); + } + return defaultValue(method); + }); + } + + protected static void deliver(Channel channel, String body) throws Exception { + AMQPUrlReceiver receiver = new AMQPUrlReceiver(); + receiver.new UrlConsumer(channel).handleDelivery( + "amq.ctag-test", envelope(), null, body.getBytes("UTF-8")); + } + + /** + * A failed ack must not escape. The broker redelivers, so swallowing it + * costs a duplicate; letting it out costs the channel. + */ + @Test + public void testAckFailureOnClosedChannelDoesNotEscape() throws Exception { + AtomicBoolean acked = new AtomicBoolean(); + + // Not a GET, so the handler skips processing and goes to the ack. + deliver(closedChannelRecording(acked), + "{\"url\": \"https://example.org/\", \"method\": \"POST\"}"); + + assertTrue(acked.get(), "the ack should have been attempted"); + } + + /** + * Malformed json is a poison message: parsing it used to run outside the + * try and kill the channel for every other consumer on it. + */ + @Test + public void testMalformedJsonDoesNotEscape() throws Exception { + AtomicBoolean acked = new AtomicBoolean(); + + deliver(closedChannelRecording(acked), "this is not json"); + + assertTrue(acked.get(), "a poison message should still be acked, not redelivered forever"); + } + + /** A message with no method at all is ignored, not thrown on. */ + @Test + public void testMissingMethodDoesNotEscape() throws Exception { + AtomicBoolean acked = new AtomicBoolean(); + + deliver(closedChannelRecording(acked), "{\"url\": \"https://example.org/\"}"); + + assertTrue(acked.get()); + } + + @SuppressWarnings("unchecked") + protected static AtomicReference consumerTagOf(AMQPUrlReceiver receiver) + throws Exception { + Field field = AMQPUrlReceiver.class.getDeclaredField("consumerTag"); + field.setAccessible(true); + return (AtomicReference) field.get(receiver); + } + + /** + * A late shutdown signal must not clear a tag belonging to a consumer that + * has since replaced it, or StarterRestarter starts a second consumer + * alongside the live one and they accumulate on every failure. + */ + @Test + public void testShutdownSignalLeavesANewerConsumerTagAlone() throws Exception { + AMQPUrlReceiver receiver = new AMQPUrlReceiver(); + AtomicReference tag = consumerTagOf(receiver); + tag.set("amq.ctag-new"); + + ShutdownSignalException sig = new ShutdownSignalException(false, true, null, null); + receiver.new UrlConsumer(null).handleShutdownSignal("amq.ctag-old", sig); + + assertEquals("amq.ctag-new", tag.get()); + } + + /** The matching signal does still clear it, so the restarter rebuilds. */ + @Test + public void testShutdownSignalClearsItsOwnConsumerTag() throws Exception { + AMQPUrlReceiver receiver = new AMQPUrlReceiver(); + AtomicReference tag = consumerTagOf(receiver); + tag.set("amq.ctag-live"); + + ShutdownSignalException sig = new ShutdownSignalException(false, true, null, null); + receiver.new UrlConsumer(null).handleShutdownSignal("amq.ctag-live", sig); + + assertNull(tag.get()); + } + + /** + * Dropping the reference alone leaves the old channel live and consuming, + * invisible to this bean, since the client would go on recovering it. + */ + @Test + public void testDeadChannelIsAbortedBeforeBeingReplaced() throws Exception { + AtomicBoolean aborted = new AtomicBoolean(); + Channel dead = channelProxy((proxy, method, args) -> { + switch (method.getName()) { + case "isOpen": return false; + case "abort": aborted.set(true); return null; + default: return defaultValue(method); + } + }); + Channel fresh = channelProxy((proxy, method, args) -> + "isOpen".equals(method.getName()) ? true : defaultValue(method)); + Connection connection = connectionProxy((proxy, method, args) -> { + switch (method.getName()) { + case "isOpen": return true; + case "createChannel": return fresh; + default: return defaultValue(method); + } + }); + + AMQPUrlReceiver receiver = new AMQPUrlReceiver(); + receiver.connection = connection; + receiver.channel = dead; + + assertSame(fresh, receiver.channel()); + assertTrue(aborted.get(), "the dead channel should have been aborted"); + } + + /** The same, one level up: an unaborted connection keeps its consumers. */ + @Test + public void testDeadConnectionIsAbortedBeforeBeingReplaced() throws Exception { + AtomicBoolean aborted = new AtomicBoolean(); + Connection dead = connectionProxy((proxy, method, args) -> { + switch (method.getName()) { + case "isOpen": return false; + case "abort": aborted.set(true); return null; + default: return defaultValue(method); + } + }); + + AMQPUrlReceiver receiver = new AMQPUrlReceiver(); + receiver.setAmqpUri("amqp://guest:guest@127.0.0.1:1/%2f"); + receiver.connection = dead; + + // The replacement attempt fails -- nothing is listening -- but the + // abort must already have happened by then. + assertThrows(Exception.class, () -> receiver.connection()); + assertTrue(aborted.get(), "the dead connection should have been aborted"); + } + + /** An open channel is handed back untouched. */ + @Test + public void testOpenChannelIsReused() throws Exception { + AtomicBoolean aborted = new AtomicBoolean(); + Channel open = channelProxy((proxy, method, args) -> { + switch (method.getName()) { + case "isOpen": return true; + case "abort": aborted.set(true); return null; + default: return defaultValue(method); + } + }); + + AMQPUrlReceiver receiver = new AMQPUrlReceiver(); + receiver.channel = open; + + assertSame(open, receiver.channel()); + assertFalse(aborted.get()); + } }