From 2c7c07ba642a3e79abf232633a2a195fd4f9704a Mon Sep 17 00:00:00 2001 From: gojomo Date: Tue, 12 May 2009 22:29:54 +0000 Subject: [PATCH] apply to h3 noah's fix [HER-1510] meta refresh url from a seed considered as a seed * LinksScoper.java (considerAsSeed) return true any seed with hop type Hop.REFER, which includes meta refreshes; also simplify the code a bit --- .../archive/crawler/postprocessor/LinksScoper.java | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/LinksScoper.java b/engine/src/main/java/org/archive/crawler/postprocessor/LinksScoper.java index db65aae2..ff2d55c3 100644 --- a/engine/src/main/java/org/archive/crawler/postprocessor/LinksScoper.java +++ b/engine/src/main/java/org/archive/crawler/postprocessor/LinksScoper.java @@ -254,17 +254,8 @@ public class LinksScoper extends Scoper implements PostProcessor { private boolean considerAsSeed(final CrawlURI curi, final Link wref, final boolean redirectsNewSeeds) { - // Check if this is a seed with a 301 or 302. - if (curi.isSeed() - && (curi.getFetchStatus() == 301 || - curi.getFetchStatus() == 302) - && wref.getHopType() == Hop.REFER) { - // Check if redirects from seeds should be treated as seeds. - if (redirectsNewSeeds) { - return true; - } - } - return false; + return redirectsNewSeeds && curi.isSeed() + && wref.getHopType() == Hop.REFER; } /**