From b7d1a79b4dceec1577ccf3ed88ed42415c240d36 Mon Sep 17 00:00:00 2001 From: gojomo Date: Mon, 5 Oct 2009 23:08:12 +0000 Subject: [PATCH] More ProcessorURI -> CrawlURI updates --- .../HighestUriQueuePrecedencePolicy.java | 6 +-- .../precedence/PrecedenceProvider.java | 6 +-- .../postprocessor/LowDiskPauseProcessor.java | 27 ++++++++---- .../SupplementaryLinksScoper.java | 42 ++++++++----------- .../archive/crawler/prefetch/Preselector.java | 39 ++++++++--------- 5 files changed, 58 insertions(+), 62 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/HighestUriQueuePrecedencePolicy.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/HighestUriQueuePrecedencePolicy.java index d173d9d7..72ceb01f 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/precedence/HighestUriQueuePrecedencePolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/HighestUriQueuePrecedencePolicy.java @@ -28,7 +28,6 @@ import java.io.PrintWriter; import org.archive.crawler.frontier.WorkQueue; import org.archive.modules.CrawlURI; -import org.archive.modules.ProcessorURI; import org.archive.modules.fetcher.FetchStats.Stage; import org.archive.util.Histotable; @@ -66,11 +65,10 @@ public class HighestUriQueuePrecedencePolicy extends BaseQueuePrecedencePolicy { } /* (non-Javadoc) - * @see org.archive.crawler.frontier.precedence.PrecedenceProvider#tally(org.archive.modules.ProcessorURI, org.archive.modules.fetcher.FetchStats.Stage) + * @see org.archive.crawler.frontier.precedence.PrecedenceProvider#tally(org.archive.modules.CrawlURI, org.archive.modules.fetcher.FetchStats.Stage) */ @Override - public void tally(ProcessorURI puri, Stage stage) { - CrawlURI curi = (CrawlURI)puri; + public void tally(CrawlURI curi, Stage stage) { switch(stage) { case SCHEDULED: // enqueued diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceProvider.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceProvider.java index d8e4f0c9..a931aa8c 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceProvider.java +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceProvider.java @@ -27,7 +27,7 @@ package org.archive.crawler.frontier.precedence; import java.io.PrintWriter; import java.io.Serializable; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.modules.fetcher.FetchStats; import org.archive.modules.fetcher.FetchStats.Stage; import org.archive.util.ArchiveUtils; @@ -43,9 +43,9 @@ FetchStats.CollectsFetchStats, Serializable { abstract public int getPrecedence(); /* (non-Javadoc) - * @see org.archive.modules.fetcher.FetchStats.CollectsFetchStats#tally(org.archive.modules.ProcessorURI, org.archive.modules.fetcher.FetchStats.Stage) + * @see org.archive.modules.fetcher.FetchStats.CollectsFetchStats#tally(org.archive.modules.CrawlURI, org.archive.modules.fetcher.FetchStats.Stage) */ - public void tally(ProcessorURI curi, Stage stage) { + public void tally(CrawlURI curi, Stage stage) { // by default do nothing; subclasses do more } diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/LowDiskPauseProcessor.java b/engine/src/main/java/org/archive/crawler/postprocessor/LowDiskPauseProcessor.java index ec4571b2..1801305f 100644 --- a/engine/src/main/java/org/archive/crawler/postprocessor/LowDiskPauseProcessor.java +++ b/engine/src/main/java/org/archive/crawler/postprocessor/LowDiskPauseProcessor.java @@ -34,10 +34,11 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.io.IOUtils; -import org.archive.modules.PostProcessor; +import org.archive.crawler.framework.CrawlController; import org.archive.modules.ProcessResult; import org.archive.modules.Processor; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; +import org.springframework.beans.factory.annotation.Autowired; /** * Processor module which uses 'df -k', where available and with @@ -45,7 +46,7 @@ import org.archive.modules.ProcessorURI; * disk space and pause the crawl if free space on monitored * filesystems falls below certain thresholds. */ -public class LowDiskPauseProcessor extends Processor implements PostProcessor { +public class LowDiskPauseProcessor extends Processor { private static final long serialVersionUID = 3L; @@ -56,6 +57,15 @@ public class LowDiskPauseProcessor extends Processor implements PostProcessor { Logger.getLogger(LowDiskPauseProcessor.class.getName()); + protected CrawlController controller; + public CrawlController getCrawlController() { + return this.controller; + } + @Autowired + public void setCrawlController(CrawlController controller) { + this.controller = controller; + } + /** * List of filessystem mounts whose 'available' space should be monitored * via 'df' (if available). @@ -107,12 +117,12 @@ public class LowDiskPauseProcessor extends Processor implements PostProcessor { @Override - protected boolean shouldProcess(ProcessorURI curi) { + protected boolean shouldProcess(CrawlURI curi) { return true; } @Override - protected void innerProcess(ProcessorURI uri) { + protected void innerProcess(CrawlURI uri) { throw new AssertionError(); } @@ -125,7 +135,7 @@ public class LowDiskPauseProcessor extends Processor implements PostProcessor { * @param curi CrawlURI to process. */ @Override - protected ProcessResult innerProcessResult(ProcessorURI curi) { + protected ProcessResult innerProcessResult(CrawlURI curi) { synchronized (this) { contentSinceCheck += curi.getContentSize(); if (contentSinceCheck/1024 > getRecheckThresholdKb()) { @@ -145,7 +155,7 @@ public class LowDiskPauseProcessor extends Processor implements PostProcessor { * crawl pause. * @param curi Current context. */ - private ProcessResult checkAvailableSpace(ProcessorURI curi) { + private ProcessResult checkAvailableSpace(CrawlURI curi) { try { String df = IOUtils.toString(Runtime.getRuntime().exec( "df -k").getInputStream()); @@ -166,7 +176,8 @@ public class LowDiskPauseProcessor extends Processor implements PostProcessor { availKilobytes + "K available on " + mount + " (below threshold " + thresholdKilobytes + "K)"); - return ProcessResult.STUCK; + controller.requestCrawlPause(); + return ProcessResult.PROCEED; } } } diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java b/engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java index b8a4ac10..03608d18 100644 --- a/engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java +++ b/engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java @@ -1,28 +1,22 @@ -/* SupplementaryLinksScoper - * - * $Id$ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * Created on Oct 2, 2003 - * - * Copyright (C) 2003 Internet Archive. + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * This file is part of the Heritrix web crawler (crawler.archive.org). + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ + package org.archive.crawler.postprocessor; import java.util.Iterator; @@ -31,8 +25,6 @@ import java.util.logging.Logger; import org.archive.crawler.framework.Scoper; import org.archive.modules.CrawlURI; -import org.archive.modules.PostProcessor; -import org.archive.modules.ProcessorURI; import org.archive.modules.deciderules.AcceptDecideRule; import org.archive.modules.deciderules.DecideResult; import org.archive.modules.deciderules.DecideRule; @@ -49,7 +41,7 @@ import org.archive.modules.deciderules.DecideRule; * to handle rejected URLs otherwise. * @author stack */ -public class SupplementaryLinksScoper extends Scoper implements PostProcessor { +public class SupplementaryLinksScoper extends Scoper { private static final long serialVersionUID = -3L; @@ -80,12 +72,12 @@ public class SupplementaryLinksScoper extends Scoper implements PostProcessor { } - protected boolean shouldProcess(ProcessorURI puri) { + protected boolean shouldProcess(CrawlURI puri) { return puri instanceof CrawlURI; } - protected void innerProcess(final ProcessorURI puri) { + protected void innerProcess(final CrawlURI puri) { CrawlURI curi = (CrawlURI)puri; // If prerequisites or no links, nothing to be done in here. diff --git a/engine/src/main/java/org/archive/crawler/prefetch/Preselector.java b/engine/src/main/java/org/archive/crawler/prefetch/Preselector.java index 31e3940b..9457ce11 100644 --- a/engine/src/main/java/org/archive/crawler/prefetch/Preselector.java +++ b/engine/src/main/java/org/archive/crawler/prefetch/Preselector.java @@ -1,26 +1,22 @@ -/* Copyright (C) 2003 Internet Archive. +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * This file is part of the Heritrix web crawler (crawler.archive.org). + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. + * http://www.apache.org/licenses/LICENSE-2.0 * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * SimplePreselector.java - * Created on Sep 22, 2003 - * - * $Header$ + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ + package org.archive.crawler.prefetch; @@ -30,7 +26,6 @@ import static org.archive.modules.fetcher.FetchStatusCodes.*; import org.archive.crawler.framework.Scoper; import org.archive.modules.CrawlURI; import org.archive.modules.ProcessResult; -import org.archive.modules.ProcessorURI; import org.archive.util.TextUtils; @@ -112,19 +107,19 @@ public class Preselector extends Scoper { } @Override - protected boolean shouldProcess(ProcessorURI puri) { + protected boolean shouldProcess(CrawlURI puri) { return puri instanceof CrawlURI; } @Override - protected void innerProcess(ProcessorURI puri) { + protected void innerProcess(CrawlURI puri) { throw new AssertionError(); } @Override - protected ProcessResult innerProcessResult(ProcessorURI puri) { + protected ProcessResult innerProcessResult(CrawlURI puri) { CrawlURI curi = (CrawlURI)puri; // Check if uris should be blocked