From eaec682dd4b2f382cad7e40bc7df77bd2746ae72 Mon Sep 17 00:00:00 2001
From: Andrew Jackson <Andrew.Jackson@bl.uk>
Date: Sun, 13 Mar 2016 20:22:27 +0000
Subject: [PATCH] Bump incoming links to MEDIUM to compete with e.g. runs of
 redirects.

---
 .../org/archive/crawler/frontier/AMQPUrlReceiver.java     | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java b/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java
index 3afd7a35..6cf96791 100644
--- a/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java
+++ b/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java
@@ -446,7 +446,13 @@ public class AMQPUrlReceiver
                 curi.setSchedulingDirective(SchedulingConstants.HIGH);
                 curi.setPrecedence(1);
             } else {
-                curi.setSchedulingDirective(SchedulingConstants.NORMAL);
+                /*
+                 * By default, redirects get set to MEDIUM, so to ensure the
+                 * incoming links are caught promptly, even if the crawler hits
+                 * a lot of redirects (which we have observed for e.g. The
+                 * Guardian), we bump them to MEDIUM so these links can compete.
+                 */
+                curi.setSchedulingDirective(SchedulingConstants.MEDIUM);
                 curi.setPrecedence(1);
             }