lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-26 15:46:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

* (many)

Browse Source 
harmonize our usage on '[Rr]egex' as the abbrieviation for regular-expression
    (replacing RegExp/etc.; 'regex' is 30-40% more prevalent by google hits)
This commit is contained in:
gojomo
2009-10-06 03:47:38 +00:00
parent d86f5f7403
commit 3654e330cc
17 changed files with 31 additions and 559 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/deciderules/ContentTypeMatchesRegExpDecideRule.java
-41
View File
@@ -1,41 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.deciderules;
import org.archive.modules.CrawlURI;
/**
* DecideRule whose decision is applied if the URI's content-type
* is present and matches the supplied regular expression.
*
* @author Olaf Freyer
*/
public class ContentTypeMatchesRegExpDecideRule extends MatchesRegExpDecideRule{
private static final long serialVersionUID = -2066930281015155843L;
public ContentTypeMatchesRegExpDecideRule() {
}
@Override
protected String getString(CrawlURI uri) {
return uri.getContentType();
}
}
modules/src/main/java/org/archive/modules/deciderules/ContentTypeNotMatchesRegExpDecideRule.java
-48
View File
@@ -1,48 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.deciderules;
import org.archive.modules.CrawlURI;
/**
* DecideRule whose decision is applied if the URI's content-type
* is present and does not match the supplied regular expression.
*
* @author Olaf Freyer
*/
public class ContentTypeNotMatchesRegExpDecideRule extends
ContentTypeMatchesRegExpDecideRule {
private static final long serialVersionUID = 4729800377757426137L;
public ContentTypeNotMatchesRegExpDecideRule() {
}
/**
* Evaluate whether given object's string version does not match
* configured regexp (by reversing the superclass's answer).
*
* @param object Object to make decision about.
* @return true if the regexp is not matched
*/
@Override
protected boolean evaluate(CrawlURI o) {
return !super.evaluate(o);
}
}
modules/src/main/java/org/archive/modules/deciderules/FetchStatusMatchesRegExpDecideRule.java
-41
View File
@@ -1,41 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.deciderules;
import org.archive.modules.CrawlURI;
public class FetchStatusMatchesRegExpDecideRule extends MatchesRegExpDecideRule {
private static final long serialVersionUID = 3L;
/**
* Usual constructor.
*/
public FetchStatusMatchesRegExpDecideRule() {
}
protected String getString(CrawlURI uri) {
return Integer.toString(uri.getFetchStatus());
}
}
modules/src/main/java/org/archive/modules/deciderules/FetchStatusNotMatchesRegExpDecideRule.java
-48
View File
@@ -1,48 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.deciderules;
import org.archive.modules.CrawlURI;
public class FetchStatusNotMatchesRegExpDecideRule
extends FetchStatusMatchesRegExpDecideRule {
private static final long serialVersionUID = -2220182698344063577L;
// private final Logger logger = Logger.getLogger(this.getClass().getName());
/**
* Usual constructor.
* @param name
*/
public FetchStatusNotMatchesRegExpDecideRule() {
}
/**
* Evaluate whether given object's FetchStatus does not match
* configured regexp (by reversing the superclass's answer).
*
* @param object Object to make decision about.
* @return true if the regexp is not matched
*/
@Override
protected boolean evaluate(CrawlURI object) {
return ! super.evaluate(object);
}
}
modules/src/main/java/org/archive/modules/deciderules/HopsPathMatchesRegExpDecideRule.java
-44
View File
@@ -1,44 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.deciderules;
import org.archive.modules.CrawlURI;
/**
* Rule applies configured decision to any CrawlURIs whose 'hops-path'
* (string like "LLXE" etc.) matches the supplied regexp.
*
* @author gojomo
*/
public class HopsPathMatchesRegExpDecideRule extends MatchesRegExpDecideRule {
private static final long serialVersionUID = 3L;
/**
* Usual constructor.
* @param name
*/
public HopsPathMatchesRegExpDecideRule() {
}
@Override
protected String getString(CrawlURI uri) {
return uri.getPathFromSeed();
}
}
modules/src/main/java/org/archive/modules/deciderules/MatchesListRegExpDecideRule.java
-126
View File
@@ -1,126 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.deciderules;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Pattern;
import org.archive.modules.CrawlURI;
/**
* Rule applies configured decision to any CrawlURIs whose String URI
* matches the supplied regexps.
* <p>
* The list of regular expressions can be considered logically AND or OR.
*
* @author Kristinn Sigurdsson
*
* @see MatchesRegExpDecideRule
*/
public class MatchesListRegExpDecideRule extends PredicatedDecideRule {
private static final long serialVersionUID = 3L;
private static final Logger logger =
Logger.getLogger(MatchesListRegExpDecideRule.class.getName());
/**
* The list of regular expressions to evalute against the URI.
*/
{
setRegexList(new ArrayList<Pattern>());
}
@SuppressWarnings("unchecked")
public List<Pattern> getRegexList() {
return (List<Pattern>) kp.get("regexList");
}
public void setRegexList(List<Pattern> patterns) {
kp.put("regexList", patterns);
}
/**
* True if the list of regular expression should be considered as logically
* AND when matching. False if the list of regular expressions should be
* considered as logically OR when matching.
*/
{
setListLogicalOr(true);
}
public boolean getListLogicalOr() {
return (Boolean) kp.get("listLogicalOr");
}
public void setListLogicalOr(boolean listLogicalOr) {
kp.put("listLogicalOr",listLogicalOr);
}
/**
* Usual constructor.
*/
public MatchesListRegExpDecideRule() {
}
/**
* Evaluate whether given object's string version
* matches configured regexps
*/
@Override
protected boolean evaluate(CrawlURI uri) {
List<Pattern> regexps = getRegexList();
if(regexps.size()==0){
return false;
}
String str = uri.toString();
boolean listLogicOR = getListLogicalOr();
for (Pattern p: regexps) {
boolean matches = p.matcher(str).matches();
if (logger.isLoggable(Level.FINER)) {
logger.finer("Tested '" + str + "' match with regex '" +
p.pattern() + " and result was " + matches);
}
if(matches){
if(listLogicOR){
// OR based and we just got a match, done!
logger.fine("Matched: " + str);
return true;
}
} else {
if(listLogicOR == false){
// AND based and we just found a non-match, done!
return false;
}
}
}
if (listLogicOR) {
return false;
} else {
return true;
}
}
}
modules/src/main/java/org/archive/modules/deciderules/MatchesRegExpDecideRule.java
-68
View File
@@ -1,68 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.deciderules;
import java.util.regex.Pattern;
import org.archive.modules.CrawlURI;
/**
* Rule applies configured decision to any CrawlURIs whose String URI
* matches the supplied regexp.
*
* @author gojomo
*/
public class MatchesRegExpDecideRule extends PredicatedDecideRule {
private static final long serialVersionUID = 2L;
{
setRegex(Pattern.compile("."));
}
public Pattern getRegex() {
return (Pattern) kp.get("regex");
}
public void setRegex(Pattern regex) {
kp.put("regex",regex);
}
/**
* Usual constructor.
*/
public MatchesRegExpDecideRule() {
}
/**
* Evaluate whether given object's string version
* matches configured regexp
*
* @param object
* @return true if regexp is matched
*/
@Override
protected boolean evaluate(CrawlURI uri) {
Pattern p = getRegex();
return p.matcher(getString(uri)).matches();
}
protected String getString(CrawlURI uri) {
return uri.toString();
}
}
modules/src/main/java/org/archive/modules/deciderules/NotMatchesFilePatternDecideRule.java
+4 -4
View File
@@ -23,7 +23,7 @@ import org.archive.modules.CrawlURI;
/**
* Rule applies configured decision to any URIs which do *not*
* match the supplied (file-pattern) regexp.
* match the supplied (file-pattern) regex.
*
* @author gojomo
*/
@@ -32,7 +32,7 @@ extends MatchesFilePatternDecideRule {
private static final long serialVersionUID = -8161371026787859554L;
//private static final Logger logger =
// Logger.getLogger(NotMatchesRegExpDecideRule.class.getName());
// Logger.getLogger(NotMatchesRegexDecideRule.class.getName());
/**
* Usual constructor.
@@ -43,10 +43,10 @@ extends MatchesFilePatternDecideRule {
/**
* Evaluate whether given object's string version does not match
* configured regexp (by reversing the superclass's answer).
* configured regex (by reversing the superclass's answer).
*
* @param object Object to make decision about.
* @return true if the regexp is not matched
* @return true if the regex is not matched
*/
@Override
protected boolean evaluate(CrawlURI uri) {
modules/src/main/java/org/archive/modules/deciderules/NotMatchesListRegExpDecideRule.java
-56
View File
@@ -1,56 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.deciderules;
import org.archive.modules.CrawlURI;
/**
* Rule applies configured decision to any URIs which do *not*
* match the supplied regexp.
*
* @author Kristinn Sigurdsson
*/
public class NotMatchesListRegExpDecideRule extends MatchesListRegExpDecideRule {
private static final long serialVersionUID = 8691360087063555583L;
//private static final Logger logger =
// Logger.getLogger(NotMatchesListRegExpDecideRule.class.getName());
/**
* Usual constructor.
* @param name
*/
public NotMatchesListRegExpDecideRule() {
}
/**
* Evaluate whether given object's string version does not match
* configured regexps (by reversing the superclass's answer).
*
* @param object Object to make decision about.
* @return true if the regexps are not matched
*/
@Override
protected boolean evaluate(CrawlURI object) {
return ! super.evaluate(object);
}
}
modules/src/main/java/org/archive/modules/deciderules/NotMatchesRegExpDecideRule.java
-54
View File
@@ -1,54 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.deciderules;
import org.archive.modules.CrawlURI;
/**
* Rule applies configured decision to any URIs which do *not*
* match the supplied regexp.
*
* @author gojomo
*/
public class NotMatchesRegExpDecideRule extends MatchesRegExpDecideRule {
private static final long serialVersionUID = -2085313401991694306L;
//private static final Logger logger =
// Logger.getLogger(NotMatchesRegExpDecideRule.class.getName());
/**
* Usual constructor.
*/
public NotMatchesRegExpDecideRule(String name) {
}
/**
* Evaluate whether given object's string version does not match
* configured regexp (by reversing the superclass's answer).
*
* @param object Object to make decision about.
* @return true if the regexp is not matched
*/
@Override
protected boolean evaluate(CrawlURI object) {
return ! super.evaluate(object);
}
}
modules/src/main/java/org/archive/modules/deciderules/PathologicalPathDecideRule.java
+4 -5
View File
@@ -32,7 +32,6 @@ import org.archive.modules.CrawlURI;
* @author gojomo
*/
public class PathologicalPathDecideRule extends DecideRule {
private static final long serialVersionUID = 3L;
/**
@@ -72,9 +71,9 @@ public class PathologicalPathDecideRule extends DecideRule {
}
/**
* Construct the regexp string to be matched against the URI.
* Construct the regex string to be matched against the URI.
* @param o an object to extract a URI from.
* @return the regexp pattern.
* @return the regex pattern.
*/
private Pattern getPattern(int maxRep) {
// race no concern: assignment is atomic, happy with any last value
@@ -82,13 +81,13 @@ public class PathologicalPathDecideRule extends DecideRule {
if (p != null) {
return p;
}
String regex = constructRegexp(maxRep);
String regex = constructRegex(maxRep);
p = Pattern.compile(regex);
pattern.set(p);
return p;
}
protected String constructRegexp(int rep) {
protected String constructRegex(int rep) {
return (rep == 0) ? null : ".*?/(.*?/)\\1{" + rep + ",}.*";
}
}
modules/src/main/java/org/archive/modules/deciderules/surt/NotSurtPrefixedDecideRule.java
+1 -1
View File
@@ -49,7 +49,7 @@ public class NotSurtPrefixedDecideRule extends SurtPrefixedDecideRule {
* prefix set -- simply reverse superclass's determination
*
* @param object
* @return true if regexp is matched
* @return true if regex is matched
*/
protected boolean evaluate(CrawlURI object) {
return !super.evaluate(object);
modules/src/main/java/org/archive/modules/extractor/AggressiveExtractorHTML.java
+2 -2
View File
@@ -25,7 +25,7 @@ import org.archive.modules.CrawlURI;
/**
* Extended version of ExtractorHTML with more aggressive javascript link
* extraction where javascript code is parsed first with general HTML tags
* regexp, and than by javascript speculative link regexp.
* regex, and than by javascript speculative link regex.
*
* @author Igor Ranitovic
*
@@ -57,7 +57,7 @@ extends ExtractorHTML {
ret.append(super.report());
ret.append(" Function: Link extraction on HTML documents " +
"(including embedded CSS)\n");
ret.append(" ProcessorURRIs handled: " + numberOfCURIsHandled + "\n");
ret.append(" ProcessorURIs handled: " + numberOfCURIsHandled + "\n");
ret.append(" Links extracted: " + numberOfLinksExtracted + "\n");
return ret.toString();
}
modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java
+10 -11
View File
@@ -29,6 +29,7 @@ import java.util.regex.Pattern;
import org.apache.commons.httpclient.URIException;
import org.archive.io.ReplayCharSequence;
import org.archive.modules.CrawlMetadata;
import org.archive.modules.CrawlURI;
import org.archive.modules.net.RobotsHonoringPolicy;
import org.archive.net.UURI;
@@ -287,26 +288,24 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
}
/**
* The robots honoring policy to use when considering a robots META tag.
* CrawlMetadata provides the robots honoring policy to use when
* considering a robots META tag.
*/
public RobotsHonoringPolicy getRobotsHonoringPolicy() {
return (RobotsHonoringPolicy) kp.get("robotsHonoringPolicy");
CrawlMetadata metadata;
public CrawlMetadata getMetadata() {
return metadata;
}
@Autowired
public void setRobotsHonoringPolicy(RobotsHonoringPolicy policy) {
kp.put("robotsHonoringPolicy",policy);
public void setMetadata(CrawlMetadata provider) {
this.metadata = provider;
}
protected long numberOfCURIsHandled = 0;
protected long numberOfLinksExtracted = 0;
RobotsHonoringPolicy honoringPolicy;
private Pattern relevantTagExtractor;
private Pattern eachAttributeExtractor;
public ExtractorHTML() {
}
@@ -765,7 +764,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
// Look for the 'robots' meta-tag
if("robots".equalsIgnoreCase(name) && content != null ) {
curi.getData().put(A_META_ROBOTS, content);
RobotsHonoringPolicy policy = honoringPolicy;
RobotsHonoringPolicy policy = metadata.getRobotsHonoringPolicy();
String contentLower = content.toLowerCase();
if ((policy == null
|| (!policy.isType(RobotsHonoringPolicy.Type.IGNORE)
modules/src/main/java/org/archive/modules/extractor/HTTPContentDigest.java
+4 -4
View File
@@ -115,8 +115,8 @@ public class HTTPContentDigest extends Processor {
protected void innerProcess(CrawlURI curi) throws InterruptedException {
// Ok, if we got this far we need to calculate the content digest.
// Get the regexpr
Pattern regexpr = getStripRegex();
// Get the regex
Pattern regex = getStripRegex();
// Get a replay of the document character seq.
ReplayCharSequence cs = null;
@@ -135,11 +135,11 @@ public class HTTPContentDigest extends Processor {
String s = null;
if (regexpr != null) {
if (regex != null) {
s = cs.toString();
} else {
// Process the document
Matcher m = regexpr.matcher(cs);
Matcher m = regex.matcher(cs);
s = m.replaceAll(" ");
}
digest.update(s.getBytes());
modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java
+1 -1
View File
@@ -289,7 +289,7 @@ public class JerichoExtractorHTML extends ExtractorHTML {
if ("robots".equals(name) && content != null) {
curi.getData().put(A_META_ROBOTS, content);
RobotsHonoringPolicy policy = honoringPolicy;
RobotsHonoringPolicy policy = metadata.getRobotsHonoringPolicy();
String contentLower = content.toLowerCase();
if ((policy == null || (!policy.isType(
RobotsHonoringPolicy.Type.IGNORE) && !policy.isType(
modules/src/main/java/org/archive/modules/seeds/TextSeedModule.java
+5 -5
View File
@@ -44,7 +44,7 @@ import org.archive.net.UURIFactory;
import org.archive.spring.WriteTarget;
import org.archive.util.DevUtils;
import org.archive.util.iterator.LineReadingIterator;
import org.archive.util.iterator.RegexpLineIterator;
import org.archive.util.iterator.RegexLineIterator;
import org.springframework.beans.factory.annotation.Required;
/**
@@ -100,11 +100,11 @@ implements ReadSource,
protected void announceSeedsFromReader(BufferedReader reader) {
String s;
Iterator<String> iter =
new RegexpLineIterator(
new RegexLineIterator(
new LineReadingIterator(reader),
RegexpLineIterator.COMMENT_LINE,
RegexpLineIterator.NONWHITESPACE_ENTRY_TRAILING_COMMENT,
RegexpLineIterator.ENTRY);
RegexLineIterator.COMMENT_LINE,
RegexLineIterator.NONWHITESPACE_ENTRY_TRAILING_COMMENT,
RegexLineIterator.ENTRY);
while (iter.hasNext()) {
s = (String) iter.next();