From e2a61792a764d7474ef17964f9b424fbdf90cf86 Mon Sep 17 00:00:00 2001 From: gojomo Date: Tue, 6 Oct 2009 04:09:49 +0000 Subject: [PATCH] * (many) harmonize our usage on '[Rr]egex' as the abbrieviation for regular-expression (replacing RegExp/etc.; 'regex' is 30-40% more prevalent by google hits) --- .../ClassKeyMatchesRegExpDecideRule.java | 66 ------------------- .../archive/crawler/framework/Frontier.java | 4 +- .../crawler/frontier/AbstractFrontier.java | 10 +-- .../prefetch/PreconditionEnforcer.java | 31 ++++----- .../archive/crawler/prefetch/Preselector.java | 12 ++-- 5 files changed, 26 insertions(+), 97 deletions(-) delete mode 100644 engine/src/main/java/org/archive/crawler/deciderules/ClassKeyMatchesRegExpDecideRule.java diff --git a/engine/src/main/java/org/archive/crawler/deciderules/ClassKeyMatchesRegExpDecideRule.java b/engine/src/main/java/org/archive/crawler/deciderules/ClassKeyMatchesRegExpDecideRule.java deleted file mode 100644 index cf33e314..00000000 --- a/engine/src/main/java/org/archive/crawler/deciderules/ClassKeyMatchesRegExpDecideRule.java +++ /dev/null @@ -1,66 +0,0 @@ -/* ClassKeyMatchesRegExpDecideRule -* -* $Id$ -* -* Created on Apr 4, 2005 -* -* Copyright (C) 2005 Internet Archive. -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ -package org.archive.crawler.deciderules; - - -import org.archive.crawler.framework.CrawlController; -import org.archive.modules.CrawlURI; -import org.archive.modules.deciderules.MatchesRegExpDecideRule; -import org.springframework.beans.factory.annotation.Autowired; - - -/** - * Rule applies configured decision to any CrawlURI class key -- i.e. - * {@link CrawlURI#getClassKey()} -- matches matches supplied regexp. - * - * @author gojomo - */ -public class ClassKeyMatchesRegExpDecideRule extends MatchesRegExpDecideRule { - - private static final long serialVersionUID = 3L; - - protected CrawlController controller; - public CrawlController getCrawlController() { - return this.controller; - } - @Autowired - public void setCrawlController(CrawlController controller) { - this.controller = controller; - } - - /** - * Usual constructor. - */ - public ClassKeyMatchesRegExpDecideRule() { - } - - - @Override - protected String getString(CrawlURI uri) { - CrawlURI curi = (CrawlURI)uri; - return controller.getFrontier().getClassKey(curi); - } - -} \ No newline at end of file diff --git a/engine/src/main/java/org/archive/crawler/framework/Frontier.java b/engine/src/main/java/org/archive/crawler/framework/Frontier.java index bef1c8ff..c4268f8c 100644 --- a/engine/src/main/java/org/archive/crawler/framework/Frontier.java +++ b/engine/src/main/java/org/archive/crawler/framework/Frontier.java @@ -348,7 +348,7 @@ public interface Frontier extends Lifecycle, MultiReporter { /** * Get a URIFrontierMarker initialized with the given * regular expression at the 'start' of the Frontier. - * @param regexpr The regular expression that URIs within the frontier must + * @param regex The regular expression that URIs within the frontier must * match to be considered within the scope of this marker * @param inCacheOnly If set to true, only those URIs within the frontier * that are stored in cache (usually this means in memory @@ -359,7 +359,7 @@ public interface Frontier extends Lifecycle, MultiReporter { * @return A URIFrontierMarker that is set for the 'start' of the frontier's * URI list. */ -// public FrontierMarker getInitialMarker(String regexpr, +// public FrontierMarker getInitialMarker(String regex, // boolean inCacheOnly); /** diff --git a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java index c5161980..33149df0 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java @@ -78,7 +78,7 @@ import org.archive.spring.KeyedProperties; import org.archive.util.ArchiveUtils; import org.archive.util.IoUtils; import org.archive.util.iterator.LineReadingIterator; -import org.archive.util.iterator.RegexpLineIterator; +import org.archive.util.iterator.RegexLineIterator; import org.json.JSONException; import org.json.JSONObject; import org.springframework.beans.factory.annotation.Autowired; @@ -1048,8 +1048,8 @@ public abstract class AbstractFrontier output = "$1"; } else { extractor = - RegexpLineIterator.NONWHITESPACE_ENTRY_TRAILING_COMMENT; - output = RegexpLineIterator.ENTRY; + RegexLineIterator.NONWHITESPACE_ENTRY_TRAILING_COMMENT; + output = RegexLineIterator.ENTRY; } // Read the input stream. @@ -1061,8 +1061,8 @@ public abstract class AbstractFrontier DecideRule scope = scopeScheduleds ? getScope() : null; try { br = new BufferedReader(new InputStreamReader(new FileInputStream(path))); - Iterator iter = new RegexpLineIterator(new LineReadingIterator(br), - RegexpLineIterator.COMMENT_LINE, extractor, output); + Iterator iter = new RegexLineIterator(new LineReadingIterator(br), + RegexLineIterator.COMMENT_LINE, extractor, output); while(iter.hasNext()) { try { diff --git a/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java b/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java index 8bccbfb1..5a25a18e 100644 --- a/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java +++ b/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java @@ -1,25 +1,20 @@ -/* Copyright (C) 2003 Internet Archive. +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * This file is part of the Heritrix web crawler (crawler.archive.org). + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. + * http://www.apache.org/licenses/LICENSE-2.0 * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * SimplePolitenessEnforcer.java - * Created on May 22, 2003 - * - * $Header$ + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.crawler.prefetch; diff --git a/engine/src/main/java/org/archive/crawler/prefetch/Preselector.java b/engine/src/main/java/org/archive/crawler/prefetch/Preselector.java index 9457ce11..f687d9ae 100644 --- a/engine/src/main/java/org/archive/crawler/prefetch/Preselector.java +++ b/engine/src/main/java/org/archive/crawler/prefetch/Preselector.java @@ -129,18 +129,18 @@ public class Preselector extends Scoper { } // Check if allowed by regular expression - String regexp = getAllowByRegex(); - if (regexp != null && !regexp.equals("")) { - if (!TextUtils.matches(regexp, curi.toString())) { + String regex = getAllowByRegex(); + if (regex != null && !regex.equals("")) { + if (!TextUtils.matches(regex, curi.toString())) { curi.setFetchStatus(S_BLOCKED_BY_USER); return ProcessResult.FINISH; } } // Check if blocked by regular expression - regexp = getBlockByRegex(); - if (regexp != null && !regexp.equals("")) { - if (TextUtils.matches(regexp, curi.toString())) { + regex = getBlockByRegex(); + if (regex != null && !regex.equals("")) { + if (TextUtils.matches(regex, curi.toString())) { curi.setFetchStatus(S_BLOCKED_BY_USER); return ProcessResult.FINISH; }