mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-09-25 23:25:44 +00:00
* (many)
harmonize our usage on '[Rr]egex' as the abbrieviation for regular-expression
(replacing RegExp/etc.; 'regex' is 30-40% more prevalent by google hits)
This commit is contained in:
gojomo
-66
@@ -1,66 +0,0 @@
|
||||
/* ClassKeyMatchesRegExpDecideRule
|
||||
*
|
||||
* $Id$
|
||||
*
|
||||
* Created on Apr 4, 2005
|
||||
*
|
||||
* Copyright (C) 2005 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*/
|
||||
package org.archive.crawler.deciderules;
|
||||
|
||||
|
||||
import org.archive.crawler.framework.CrawlController;
|
||||
import org.archive.modules.CrawlURI;
|
||||
import org.archive.modules.deciderules.MatchesRegExpDecideRule;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
||||
|
||||
/**
|
||||
* Rule applies configured decision to any CrawlURI class key -- i.e.
|
||||
* {@link CrawlURI#getClassKey()} -- matches matches supplied regexp.
|
||||
*
|
||||
* @author gojomo
|
||||
*/
|
||||
public class ClassKeyMatchesRegExpDecideRule extends MatchesRegExpDecideRule {
|
||||
|
||||
private static final long serialVersionUID = 3L;
|
||||
|
||||
protected CrawlController controller;
|
||||
public CrawlController getCrawlController() {
|
||||
return this.controller;
|
||||
}
|
||||
@Autowired
|
||||
public void setCrawlController(CrawlController controller) {
|
||||
this.controller = controller;
|
||||
}
|
||||
|
||||
/**
|
||||
* Usual constructor.
|
||||
*/
|
||||
public ClassKeyMatchesRegExpDecideRule() {
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected String getString(CrawlURI uri) {
|
||||
CrawlURI curi = (CrawlURI)uri;
|
||||
return controller.getFrontier().getClassKey(curi);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -348,7 +348,7 @@ public interface Frontier extends Lifecycle, MultiReporter {
|
||||
/**
|
||||
* Get a <code>URIFrontierMarker</code> initialized with the given
|
||||
* regular expression at the 'start' of the Frontier.
|
||||
* @param regexpr The regular expression that URIs within the frontier must
|
||||
* @param regex The regular expression that URIs within the frontier must
|
||||
* match to be considered within the scope of this marker
|
||||
* @param inCacheOnly If set to true, only those URIs within the frontier
|
||||
* that are stored in cache (usually this means in memory
|
||||
@@ -359,7 +359,7 @@ public interface Frontier extends Lifecycle, MultiReporter {
|
||||
* @return A URIFrontierMarker that is set for the 'start' of the frontier's
|
||||
* URI list.
|
||||
*/
|
||||
// public FrontierMarker getInitialMarker(String regexpr,
|
||||
// public FrontierMarker getInitialMarker(String regex,
|
||||
// boolean inCacheOnly);
|
||||
|
||||
/**
|
||||
|
||||
@@ -78,7 +78,7 @@ import org.archive.spring.KeyedProperties;
|
||||
import org.archive.util.ArchiveUtils;
|
||||
import org.archive.util.IoUtils;
|
||||
import org.archive.util.iterator.LineReadingIterator;
|
||||
import org.archive.util.iterator.RegexpLineIterator;
|
||||
import org.archive.util.iterator.RegexLineIterator;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@@ -1048,8 +1048,8 @@ public abstract class AbstractFrontier
|
||||
output = "$1";
|
||||
} else {
|
||||
extractor =
|
||||
RegexpLineIterator.NONWHITESPACE_ENTRY_TRAILING_COMMENT;
|
||||
output = RegexpLineIterator.ENTRY;
|
||||
RegexLineIterator.NONWHITESPACE_ENTRY_TRAILING_COMMENT;
|
||||
output = RegexLineIterator.ENTRY;
|
||||
}
|
||||
|
||||
// Read the input stream.
|
||||
@@ -1061,8 +1061,8 @@ public abstract class AbstractFrontier
|
||||
DecideRule scope = scopeScheduleds ? getScope() : null;
|
||||
try {
|
||||
br = new BufferedReader(new InputStreamReader(new FileInputStream(path)));
|
||||
Iterator<String> iter = new RegexpLineIterator(new LineReadingIterator(br),
|
||||
RegexpLineIterator.COMMENT_LINE, extractor, output);
|
||||
Iterator<String> iter = new RegexLineIterator(new LineReadingIterator(br),
|
||||
RegexLineIterator.COMMENT_LINE, extractor, output);
|
||||
while(iter.hasNext()) {
|
||||
try {
|
||||
|
||||
|
||||
@@ -1,25 +1,20 @@
|
||||
/* Copyright (C) 2003 Internet Archive.
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*
|
||||
* SimplePolitenessEnforcer.java
|
||||
* Created on May 22, 2003
|
||||
*
|
||||
* $Header$
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.archive.crawler.prefetch;
|
||||
|
||||
|
||||
@@ -129,18 +129,18 @@ public class Preselector extends Scoper {
|
||||
}
|
||||
|
||||
// Check if allowed by regular expression
|
||||
String regexp = getAllowByRegex();
|
||||
if (regexp != null && !regexp.equals("")) {
|
||||
if (!TextUtils.matches(regexp, curi.toString())) {
|
||||
String regex = getAllowByRegex();
|
||||
if (regex != null && !regex.equals("")) {
|
||||
if (!TextUtils.matches(regex, curi.toString())) {
|
||||
curi.setFetchStatus(S_BLOCKED_BY_USER);
|
||||
return ProcessResult.FINISH;
|
||||
}
|
||||
}
|
||||
|
||||
// Check if blocked by regular expression
|
||||
regexp = getBlockByRegex();
|
||||
if (regexp != null && !regexp.equals("")) {
|
||||
if (TextUtils.matches(regexp, curi.toString())) {
|
||||
regex = getBlockByRegex();
|
||||
if (regex != null && !regex.equals("")) {
|
||||
if (TextUtils.matches(regex, curi.toString())) {
|
||||
curi.setFetchStatus(S_BLOCKED_BY_USER);
|
||||
return ProcessResult.FINISH;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user