diff --git a/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java b/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java index 3afd7a35..6cf96791 100644 --- a/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java +++ b/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java @@ -446,7 +446,13 @@ public class AMQPUrlReceiver curi.setSchedulingDirective(SchedulingConstants.HIGH); curi.setPrecedence(1); } else { - curi.setSchedulingDirective(SchedulingConstants.NORMAL); + /* + * By default, redirects get set to MEDIUM, so to ensure the + * incoming links are caught promptly, even if the crawler hits + * a lot of redirects (which we have observed for e.g. The + * Guardian), we bump them to MEDIUM so these links can compete. + */ + curi.setSchedulingDirective(SchedulingConstants.MEDIUM); curi.setPrecedence(1); }