diff --git a/commons/src/main/java/org/archive/util/FileUtils.java b/commons/src/main/java/org/archive/util/FileUtils.java index c478247f..fb422073 100644 --- a/commons/src/main/java/org/archive/util/FileUtils.java +++ b/commons/src/main/java/org/archive/util/FileUtils.java @@ -297,7 +297,7 @@ public class FileUtils { * @return the newly created filter. */ public static IOFileFilter getRegexFileFilter(String regex) { - // Inner class defining the RegexpFileFilter + // Inner class defining the RegexFileFilter class RegexFileFilter implements IOFileFilter { Pattern pattern; diff --git a/commons/src/main/java/org/archive/util/SurtPrefixSet.java b/commons/src/main/java/org/archive/util/SurtPrefixSet.java index 4275cb7a..0db32b24 100644 --- a/commons/src/main/java/org/archive/util/SurtPrefixSet.java +++ b/commons/src/main/java/org/archive/util/SurtPrefixSet.java @@ -34,7 +34,7 @@ import java.util.Iterator; import org.archive.net.UURI; import org.archive.util.iterator.LineReadingIterator; -import org.archive.util.iterator.RegexpLineIterator; +import org.archive.util.iterator.RegexLineIterator; /** * Specialized TreeSet for keeping a set of String prefixes. @@ -63,11 +63,11 @@ public class SurtPrefixSet extends PrefixSet { String s; Iterator iter = - new RegexpLineIterator( + new RegexLineIterator( new LineReadingIterator(reader), - RegexpLineIterator.COMMENT_LINE, - RegexpLineIterator.NONWHITESPACE_ENTRY_TRAILING_COMMENT, - RegexpLineIterator.ENTRY); + RegexLineIterator.COMMENT_LINE, + RegexLineIterator.NONWHITESPACE_ENTRY_TRAILING_COMMENT, + RegexLineIterator.ENTRY); while (iter.hasNext()) { s = (String) iter.next(); @@ -83,11 +83,11 @@ public class SurtPrefixSet extends PrefixSet { String s; Iterator iter = - new RegexpLineIterator( + new RegexLineIterator( new LineReadingIterator(reader), - RegexpLineIterator.COMMENT_LINE, - RegexpLineIterator.NONWHITESPACE_ENTRY_TRAILING_COMMENT, - RegexpLineIterator.ENTRY); + RegexLineIterator.COMMENT_LINE, + RegexLineIterator.NONWHITESPACE_ENTRY_TRAILING_COMMENT, + RegexLineIterator.ENTRY); while (iter.hasNext()) { s = (String) iter.next(); @@ -109,11 +109,11 @@ public class SurtPrefixSet extends PrefixSet { String s; Iterator iter = - new RegexpLineIterator( + new RegexLineIterator( new LineReadingIterator(reader), - RegexpLineIterator.COMMENT_LINE, - RegexpLineIterator.NONWHITESPACE_ENTRY_TRAILING_COMMENT, - RegexpLineIterator.ENTRY); + RegexLineIterator.COMMENT_LINE, + RegexLineIterator.NONWHITESPACE_ENTRY_TRAILING_COMMENT, + RegexLineIterator.ENTRY); while (iter.hasNext()) { s = (String) iter.next(); diff --git a/commons/src/main/java/org/archive/util/iterator/RegexpLineIterator.java b/commons/src/main/java/org/archive/util/iterator/RegexLineIterator.java similarity index 68% rename from commons/src/main/java/org/archive/util/iterator/RegexpLineIterator.java rename to commons/src/main/java/org/archive/util/iterator/RegexLineIterator.java index c836d470..2103891c 100644 --- a/commons/src/main/java/org/archive/util/iterator/RegexpLineIterator.java +++ b/commons/src/main/java/org/archive/util/iterator/RegexLineIterator.java @@ -1,27 +1,22 @@ -/* LineReadingIterator -* -* $Id$ -* -* Created on Jul 27, 2004 -* -* Copyright (C) 2004 Internet Archive. -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.archive.util.iterator; import java.util.Iterator; @@ -31,7 +26,7 @@ import java.util.regex.Pattern; /** * Utility class providing an Iterator interface over line-oriented - * text input. By providing regexps indicating lines to ignore + * text input. By providing regexs indicating lines to ignore * (such as pure whitespace or comments), lines to consider input, and * what to return from the input lines (such as a whitespace-trimmed * non-whitespace token with optional trailing comment), this can @@ -42,10 +37,10 @@ import java.util.regex.Pattern; * * @author gojomo */ -public class RegexpLineIterator +public class RegexLineIterator extends TransformingIteratorWrapper { private static final Logger logger = - Logger.getLogger(RegexpLineIterator.class.getName()); + Logger.getLogger(RegexLineIterator.class.getName()); public static final String COMMENT_LINE = "\\s*(#.*)?"; public static final String NONWHITESPACE_ENTRY_TRAILING_COMMENT = @@ -60,7 +55,7 @@ extends TransformingIteratorWrapper { protected String outputTemplate = null; - public RegexpLineIterator(Iterator inner, String ignore, + public RegexLineIterator(Iterator inner, String ignore, String extract, String replace) { this.inner = inner; ignoreLine = Pattern.compile(ignore).matcher(""); diff --git a/engine/src/main/java/org/archive/crawler/deciderules/ClassKeyMatchesRegexDecideRule.java b/engine/src/main/java/org/archive/crawler/deciderules/ClassKeyMatchesRegexDecideRule.java new file mode 100644 index 00000000..99789413 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/deciderules/ClassKeyMatchesRegexDecideRule.java @@ -0,0 +1,60 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.crawler.deciderules; + + +import org.archive.crawler.framework.CrawlController; +import org.archive.modules.CrawlURI; +import org.archive.modules.deciderules.MatchesRegexDecideRule; +import org.springframework.beans.factory.annotation.Autowired; + + +/** + * Rule applies configured decision to any CrawlURI class key -- i.e. + * {@link CrawlURI#getClassKey()} -- matches matches supplied regex. + * + * @author gojomo + */ +public class ClassKeyMatchesRegexDecideRule extends MatchesRegexDecideRule { + + private static final long serialVersionUID = 3L; + + protected CrawlController controller; + public CrawlController getCrawlController() { + return this.controller; + } + @Autowired + public void setCrawlController(CrawlController controller) { + this.controller = controller; + } + + /** + * Usual constructor. + */ + public ClassKeyMatchesRegexDecideRule() { + } + + + @Override + protected String getString(CrawlURI uri) { + CrawlURI curi = (CrawlURI)uri; + return controller.getFrontier().getClassKey(curi); + } + +} \ No newline at end of file diff --git a/engine/src/main/java/org/archive/crawler/processor/LexicalCrawlMapper.java b/engine/src/main/java/org/archive/crawler/processor/LexicalCrawlMapper.java index 344b60ef..b623b59b 100644 --- a/engine/src/main/java/org/archive/crawler/processor/LexicalCrawlMapper.java +++ b/engine/src/main/java/org/archive/crawler/processor/LexicalCrawlMapper.java @@ -38,7 +38,7 @@ import org.archive.crawler.framework.Frontier; import org.archive.modules.CrawlURI; import org.archive.spring.ConfigPath; import org.archive.util.iterator.LineReadingIterator; -import org.archive.util.iterator.RegexpLineIterator; +import org.archive.util.iterator.RegexLineIterator; import org.springframework.beans.factory.annotation.Autowired; @@ -185,11 +185,11 @@ public class LexicalCrawlMapper extends CrawlMapper { } reader = new BufferedReader(reader); Iterator iter = - new RegexpLineIterator( + new RegexLineIterator( new LineReadingIterator((BufferedReader) reader), - RegexpLineIterator.COMMENT_LINE, - RegexpLineIterator.TRIMMED_ENTRY_TRAILING_COMMENT, - RegexpLineIterator.ENTRY); + RegexLineIterator.COMMENT_LINE, + RegexLineIterator.TRIMMED_ENTRY_TRAILING_COMMENT, + RegexLineIterator.ENTRY); while (iter.hasNext()) { String[] entry = ((String) iter.next()).split("\\s+"); map.put(entry[0],entry[1]); diff --git a/engine/src/main/java/org/archive/crawler/util/LogReader.java b/engine/src/main/java/org/archive/crawler/util/LogReader.java index 218acee0..a88f768c 100644 --- a/engine/src/main/java/org/archive/crawler/util/LogReader.java +++ b/engine/src/main/java/org/archive/crawler/util/LogReader.java @@ -1,20 +1,20 @@ -/* Copyright (C) 2003 Internet Archive. +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * This file is part of the Heritrix web crawler (crawler.archive.org). + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. + * http://www.apache.org/licenses/LICENSE-2.0 * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.crawler.util; @@ -212,16 +212,16 @@ public class LogReader * log/file that matches a given regular expression. * * @param aFileName The filename of the log/file - * @param regExpr The regular expression that is to be used + * @param regex The regular expression that is to be used * @return The line number (counting from 1, not zero) of the first line * that matches the given regular expression. -1 is returned if no * line matches the regular expression. -1 also is returned if * errors occur (file not found, io exception etc.) */ - public static int findFirstLineContaining(String aFileName, String regExpr) + public static int findFirstLineContaining(String aFileName, String regex) { try { - return findFirstLineContaining(new FileReader(aFileName), regExpr); + return findFirstLineContaining(new FileReader(aFileName), regex); } catch (FileNotFoundException e) { e.printStackTrace(); return -1; @@ -288,17 +288,17 @@ public class LogReader * log/file that matches a given regular expression. * * @param aFileName The filename of the log/file - * @param regExpr The regular expression that is to be used + * @param regex The regular expression that is to be used * @return The line number (counting from 1, not zero) of the first line * that matches the given regular expression. -1 is returned if no * line matches the regular expression. -1 also is returned if * errors occur (file not found, io exception etc.) */ public static int findFirstLineContainingFromSeries(String aFileName, - String regExpr) + String regex) { try { - return findFirstLineContaining(seriesReader(aFileName), regExpr); + return findFirstLineContaining(seriesReader(aFileName), regex); } catch (IOException e) { e.printStackTrace(); return -1; @@ -310,16 +310,16 @@ public class LogReader * log/file that matches a given regular expression. * * @param reader The reader of the log/file - * @param regExpr The regular expression that is to be used + * @param regex The regular expression that is to be used * @return The line number (counting from 1, not zero) of the first line * that matches the given regular expression. -1 is returned if no * line matches the regular expression. -1 also is returned if * errors occur (file not found, io exception etc.) */ public static int findFirstLineContaining(InputStreamReader reader, - String regExpr) + String regex) { - Pattern p = Pattern.compile(regExpr); + Pattern p = Pattern.compile(regex); try{ BufferedReader bf = new BufferedReader(reader, 8192); @@ -345,7 +345,7 @@ public class LogReader * possible to have each line prepended by it's line number. * * @param aFileName The filename of the log/file - * @param regExpr The regular expression that is to be used + * @param regex The regular expression that is to be used * @param addLines How many lines (in addition to the matched line) to add. * A value less then 1 will mean that only the matched line * will be included. If another matched line is hit before @@ -367,17 +367,17 @@ public class LogReader * If a PatternSyntaxException occurs, it's error message will be * returned and the informational string will be empty (not null). */ - public static String[] getByRegExpr(String aFileName, - String regExpr, + public static String[] getByRegex(String aFileName, + String regex, int addLines, boolean prependLineNumbers, int skipFirstMatches, int numberOfMatches) { try { File f = new File(aFileName); - return getByRegExpr( + return getByRegex( new FileReader(f), - regExpr, + regex, addLines, prependLineNumbers, skipFirstMatches, @@ -395,7 +395,7 @@ public class LogReader * possible to have each line prepended by it's line number. * * @param aFileName The filename of the log/file - * @param regExpr The regular expression that is to be used + * @param regex The regular expression that is to be used * @param addLines How many lines (in addition to the matched line) to add. * A value less then 1 will mean that only the matched line * will be included. If another matched line is hit before @@ -417,17 +417,17 @@ public class LogReader * If a PatternSyntaxException occurs, it's error message will be * returned and the informational string will be empty (not null). */ - public static String[] getByRegExprFromSeries(String aFileName, - String regExpr, + public static String[] getByRegexFromSeries(String aFileName, + String regex, int addLines, boolean prependLineNumbers, int skipFirstMatches, int numberOfMatches) { try { File f = new File(aFileName); - return getByRegExpr( + return getByRegex( seriesReader(aFileName), - regExpr, + regex, addLines, prependLineNumbers, skipFirstMatches, @@ -445,7 +445,7 @@ public class LogReader * possible to have each line prepended by it's line number. * * @param reader The reader of the log/file - * @param regExpr The regular expression that is to be used + * @param regex The regular expression that is to be used * @param addLines How many lines (in addition to the matched line) to add. * A value less then 1 will mean that only the matched line * will be included. If another matched line is hit before @@ -468,8 +468,8 @@ public class LogReader * If a PatternSyntaxException occurs, it's error message will be * returned and the informational string will be empty (not null). */ - public static String[] getByRegExpr(InputStreamReader reader, - String regExpr, + public static String[] getByRegex(InputStreamReader reader, + String regex, int addLines, boolean prependLineNumbers, int skipFirstMatches, @@ -479,7 +479,7 @@ public class LogReader String info = ""; try{ - Pattern p = Pattern.compile(regExpr); + Pattern p = Pattern.compile(regex); BufferedReader bf = new BufferedReader(reader, 8192); String line = null; @@ -542,7 +542,7 @@ public class LogReader * possible to have each line prepended by it's line number. * * @param aFileName The filename of the log/file - * @param regExpr The regular expression that is to be used + * @param regex The regular expression that is to be used * @param addLines Any lines following a match that begin with this * string will also be included. We will stop including new * lines once we hit the first that does not match. @@ -562,17 +562,17 @@ public class LogReader * If a PatternSyntaxException occurs, it's error message will be * returned and the informational string will be empty (not null). */ - public static String[] getByRegExpr(String aFileName, - String regExpr, + public static String[] getByRegex(String aFileName, + String regex, String addLines, boolean prependLineNumbers, int skipFirstMatches, int numberOfMatches){ try { File f = new File(aFileName); - return getByRegExpr( + return getByRegex( new FileReader(f), - regExpr, + regex, addLines, prependLineNumbers, skipFirstMatches, @@ -590,7 +590,7 @@ public class LogReader * possible to have each line prepended by it's line number. * * @param aFileName The filename of the log/file - * @param regExpr The regular expression that is to be used + * @param regex The regular expression that is to be used * @param addLines Any lines following a match that begin with this * string will also be included. We will stop including new * lines once we hit the first that does not match. @@ -610,17 +610,17 @@ public class LogReader * If a PatternSyntaxException occurs, it's error message will be * returned and the informational string will be empty (not null). */ - public static String[] getByRegExprFromSeries(String aFileName, - String regExpr, + public static String[] getByRegexFromSeries(String aFileName, + String regex, String addLines, boolean prependLineNumbers, int skipFirstMatches, int numberOfMatches){ try { File f = new File(aFileName); - return getByRegExpr( + return getByRegex( seriesReader(aFileName), - regExpr, + regex, addLines, prependLineNumbers, skipFirstMatches, @@ -638,7 +638,7 @@ public class LogReader * possible to have each line prepended by it's line number. * * @param reader The reader of the log/file - * @param regExpr The regular expression that is to be used + * @param regex The regular expression that is to be used * @param addLines Any lines following a match that begin with this * string will also be included. We will stop including new * lines once we hit the first that does not match. @@ -659,8 +659,8 @@ public class LogReader * If a PatternSyntaxException occurs, it's error message will be * returned and the informational string will be empty (not null). */ - public static String[] getByRegExpr(InputStreamReader reader, - String regExpr, + public static String[] getByRegex(InputStreamReader reader, + String regex, String addLines, boolean prependLineNumbers, int skipFirstMatches, @@ -669,7 +669,7 @@ public class LogReader StringBuffer ret = new StringBuffer(); String info = ""; try{ - Matcher m = Pattern.compile(regExpr).matcher(""); + Matcher m = Pattern.compile(regex).matcher(""); BufferedReader bf = new BufferedReader(reader, 8192); String line = null; diff --git a/engine/src/main/java/org/archive/extractor/RegexpCSSLinkExtractor.java b/engine/src/main/java/org/archive/extractor/RegexCSSLinkExtractor.java similarity index 73% rename from engine/src/main/java/org/archive/extractor/RegexpCSSLinkExtractor.java rename to engine/src/main/java/org/archive/extractor/RegexCSSLinkExtractor.java index 07073b1e..52d3096e 100644 --- a/engine/src/main/java/org/archive/extractor/RegexpCSSLinkExtractor.java +++ b/engine/src/main/java/org/archive/extractor/RegexCSSLinkExtractor.java @@ -1,27 +1,20 @@ /* - * ExtractorCSS + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * $Id$ + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * Created on Mar 29, 2005 + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Copyright (C) 2005 Internet Archive. + * http://www.apache.org/licenses/LICENSE-2.0 * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.extractor; @@ -54,10 +47,7 @@ import org.archive.util.TextUtils; * **/ -public class RegexpCSSLinkExtractor extends CharSequenceLinkExtractor { - - // private static Logger logger = - // Logger.getLogger(RegexpCSSLinkExtractor.class.getName()); +public class RegexCSSLinkExtractor extends CharSequenceLinkExtractor { private static String ESCAPED_AMP = "&"; // CSS escapes: "Parentheses, commas, whitespace characters, single @@ -100,7 +90,7 @@ public class RegexpCSSLinkExtractor extends CharSequenceLinkExtractor { return true; } } catch (StackOverflowError e) { - DevUtils.warnHandle(e, "RegexpCSSLinkExtractor StackOverflowError"); + DevUtils.warnHandle(e, "RegexCSSLinkExtractor StackOverflowError"); } return false; } @@ -112,6 +102,6 @@ public class RegexpCSSLinkExtractor extends CharSequenceLinkExtractor { } protected static CharSequenceLinkExtractor newDefaultInstance() { - return new RegexpCSSLinkExtractor(); + return new RegexCSSLinkExtractor(); } } diff --git a/engine/src/main/java/org/archive/extractor/RegexpHTMLLinkExtractor.java b/engine/src/main/java/org/archive/extractor/RegexHTMLLinkExtractor.java similarity index 93% rename from engine/src/main/java/org/archive/extractor/RegexpHTMLLinkExtractor.java rename to engine/src/main/java/org/archive/extractor/RegexHTMLLinkExtractor.java index dfdd4e4c..07e5af46 100644 --- a/engine/src/main/java/org/archive/extractor/RegexpHTMLLinkExtractor.java +++ b/engine/src/main/java/org/archive/extractor/RegexHTMLLinkExtractor.java @@ -1,25 +1,20 @@ -/* Copyright (C) 2003 Internet Archive. +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * This file is part of the Heritrix web crawler (crawler.archive.org). + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. + * http://www.apache.org/licenses/LICENSE-2.0 * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * SimpleHTMLExtractor.java - * Created on Jun 5, 2003 - * - * $Header$ + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.extractor; @@ -49,9 +44,9 @@ import org.archive.util.TextUtils; * * @author gojomo */ -public class RegexpHTMLLinkExtractor extends CharSequenceLinkExtractor { +public class RegexHTMLLinkExtractor extends CharSequenceLinkExtractor { private static Logger logger = - Logger.getLogger(RegexpHTMLLinkExtractor.class.getName()); + Logger.getLogger(RegexHTMLLinkExtractor.class.getName()); boolean honorRobots = true; boolean extractInlineCss = true; @@ -311,7 +306,7 @@ public class RegexpHTMLLinkExtractor extends CharSequenceLinkExtractor { * @param cs */ protected void processScriptCode(CharSequence cs) { - RegexpJSLinkExtractor.extract(cs, source, base, next, + RegexJSLinkExtractor.extract(cs, source, base, next, extractErrorListener); } @@ -446,7 +441,7 @@ public class RegexpHTMLLinkExtractor extends CharSequenceLinkExtractor { sequence.subSequence(0,endOfOpenTag)); // then, parse for URIs - RegexpCSSLinkExtractor.extract(sequence.subSequence(endOfOpenTag, + RegexCSSLinkExtractor.extract(sequence.subSequence(endOfOpenTag, sequence.length()), source, base, next, extractErrorListener); } @@ -460,7 +455,7 @@ public class RegexpHTMLLinkExtractor extends CharSequenceLinkExtractor { } protected static CharSequenceLinkExtractor newDefaultInstance() { - return new RegexpHTMLLinkExtractor(); + return new RegexHTMLLinkExtractor(); } } diff --git a/engine/src/main/java/org/archive/extractor/RegexpJSLinkExtractor.java b/engine/src/main/java/org/archive/extractor/RegexJSLinkExtractor.java similarity index 73% rename from engine/src/main/java/org/archive/extractor/RegexpJSLinkExtractor.java rename to engine/src/main/java/org/archive/extractor/RegexJSLinkExtractor.java index 0a77031c..f1330244 100644 --- a/engine/src/main/java/org/archive/extractor/RegexpJSLinkExtractor.java +++ b/engine/src/main/java/org/archive/extractor/RegexJSLinkExtractor.java @@ -1,25 +1,20 @@ -/* Copyright (C) 2003 Internet Archive. +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * This file is part of the Heritrix web crawler (crawler.archive.org). + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. + * http://www.apache.org/licenses/LICENSE-2.0 * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * Created on Nov 17, 2003 - * - * To change the template for this generated file go to - * Window>Preferences>Java>Code Generation>Code and Comments + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.extractor; @@ -42,9 +37,7 @@ import org.archive.util.TextUtils; * * @author gojomo */ -public class RegexpJSLinkExtractor extends CharSequenceLinkExtractor { -// private static Logger logger = -// Logger.getLogger(RegexpJSLinkExtractor.class.getName()); +public class RegexJSLinkExtractor extends CharSequenceLinkExtractor { static final String AMP = "&"; static final String ESCAPED_AMP = "&"; @@ -109,6 +102,6 @@ public class RegexpJSLinkExtractor extends CharSequenceLinkExtractor { } protected static CharSequenceLinkExtractor newDefaultInstance() { - return new RegexpJSLinkExtractor(); + return new RegexJSLinkExtractor(); } } diff --git a/modules/src/main/java/org/archive/modules/deciderules/ContentTypeMatchesRegexDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/ContentTypeMatchesRegexDecideRule.java new file mode 100644 index 00000000..e2460e84 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/ContentTypeMatchesRegexDecideRule.java @@ -0,0 +1,41 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.deciderules; + +import org.archive.modules.CrawlURI; + + +/** + * DecideRule whose decision is applied if the URI's content-type + * is present and matches the supplied regular expression. + * + * @author Olaf Freyer + */ +public class ContentTypeMatchesRegexDecideRule extends MatchesRegexDecideRule{ + private static final long serialVersionUID = -2066930281015155843L; + + public ContentTypeMatchesRegexDecideRule() { + } + + + @Override + protected String getString(CrawlURI uri) { + return uri.getContentType(); + } +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/ContentTypeNotMatchesRegexDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/ContentTypeNotMatchesRegexDecideRule.java new file mode 100644 index 00000000..d6581dee --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/ContentTypeNotMatchesRegexDecideRule.java @@ -0,0 +1,48 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.deciderules; + +import org.archive.modules.CrawlURI; + +/** + * DecideRule whose decision is applied if the URI's content-type + * is present and does not match the supplied regular expression. + * + * @author Olaf Freyer + */ +public class ContentTypeNotMatchesRegexDecideRule extends + ContentTypeMatchesRegexDecideRule { + private static final long serialVersionUID = 4729800377757426137L; + + public ContentTypeNotMatchesRegexDecideRule() { + } + + /** + * Evaluate whether given object's string version does not match + * configured regex (by reversing the superclass's answer). + * + * @param object Object to make decision about. + * @return true if the regex is not matched + */ + @Override + protected boolean evaluate(CrawlURI o) { + return !super.evaluate(o); + } + +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/FetchStatusMatchesRegexDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/FetchStatusMatchesRegexDecideRule.java new file mode 100644 index 00000000..9f2b53ce --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/FetchStatusMatchesRegexDecideRule.java @@ -0,0 +1,35 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.deciderules; + +import org.archive.modules.CrawlURI; + +public class FetchStatusMatchesRegexDecideRule extends MatchesRegexDecideRule { + private static final long serialVersionUID = 3L; + + /** + * Usual constructor. + */ + public FetchStatusMatchesRegexDecideRule() { + } + + protected String getString(CrawlURI uri) { + return Integer.toString(uri.getFetchStatus()); + } +} \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/deciderules/FetchStatusNotMatchesRegexDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/FetchStatusNotMatchesRegexDecideRule.java new file mode 100644 index 00000000..68963b66 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/FetchStatusNotMatchesRegexDecideRule.java @@ -0,0 +1,48 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.deciderules; + +import org.archive.modules.CrawlURI; + + +public class FetchStatusNotMatchesRegexDecideRule +extends FetchStatusMatchesRegexDecideRule { + + private static final long serialVersionUID = -2220182698344063577L; +// private final Logger logger = Logger.getLogger(this.getClass().getName()); + + /** + * Usual constructor. + * @param name + */ + public FetchStatusNotMatchesRegexDecideRule() { + } + + /** + * Evaluate whether given object's FetchStatus does not match + * configured regex (by reversing the superclass's answer). + * + * @param object Object to make decision about. + * @return true if the regex is not matched + */ + @Override + protected boolean evaluate(CrawlURI object) { + return ! super.evaluate(object); + } +} \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/deciderules/HopsPathMatchesRegexDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/HopsPathMatchesRegexDecideRule.java new file mode 100644 index 00000000..68424850 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/HopsPathMatchesRegexDecideRule.java @@ -0,0 +1,44 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.deciderules; + +import org.archive.modules.CrawlURI; + +/** + * Rule applies configured decision to any CrawlURIs whose 'hops-path' + * (string like "LLXE" etc.) matches the supplied regex. + * + * @author gojomo + */ +public class HopsPathMatchesRegexDecideRule extends MatchesRegexDecideRule { + private static final long serialVersionUID = 3L; + + /** + * Usual constructor. + * @param name + */ + public HopsPathMatchesRegexDecideRule() { + } + + + @Override + protected String getString(CrawlURI uri) { + return uri.getPathFromSeed(); + } +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/MatchesFilePatternDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/MatchesFilePatternDecideRule.java index 33bea377..61377842 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/MatchesFilePatternDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/MatchesFilePatternDecideRule.java @@ -33,11 +33,11 @@ import java.util.regex.Pattern; * expression pattern, applying its configured decision to all matches. * * Several predefined patterns are available for convenience. Choosing - * 'custom' makes this the same as a regular MatchesRegExpDecideRule. + * 'custom' makes this the same as a regular MatchesRegexDecideRule. * * @author Igor Ranitovic */ -public class MatchesFilePatternDecideRule extends MatchesRegExpDecideRule { +public class MatchesFilePatternDecideRule extends MatchesRegexDecideRule { public static enum Preset { @@ -57,11 +57,11 @@ public class MatchesFilePatternDecideRule extends MatchesRegExpDecideRule { final private Pattern pattern; - Preset(String regexp) { - if (regexp == null) { + Preset(String regex) { + if (regex == null) { pattern = null; } else { - pattern = Pattern.compile(regexp); + pattern = Pattern.compile(regex); } } @@ -91,9 +91,7 @@ public class MatchesFilePatternDecideRule extends MatchesRegExpDecideRule { /** * Use a preset if configured to do so. * @param o Context - * @return Regex to use. - * - * @see org.archive.crawler.filter.URIRegExpFilter#getRegexp(Object) + * @return regex to use. */ @Override public Pattern getRegex() { diff --git a/modules/src/main/java/org/archive/modules/deciderules/MatchesListRegexDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/MatchesListRegexDecideRule.java new file mode 100644 index 00000000..1344b30b --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/MatchesListRegexDecideRule.java @@ -0,0 +1,122 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.deciderules; + +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Pattern; + +import org.archive.modules.CrawlURI; + +/** + * Rule applies configured decision to any CrawlURIs whose String URI + * matches the supplied regexs. + *

+ * The list of regular expressions can be considered logically AND or OR. + * + * @author Kristinn Sigurdsson + * + * @see MatchesRegexDecideRule + */ +public class MatchesListRegexDecideRule extends PredicatedDecideRule { + private static final long serialVersionUID = 3L; + private static final Logger logger = + Logger.getLogger(MatchesListRegexDecideRule.class.getName()); + + /** + * The list of regular expressions to evalute against the URI. + */ + { + setRegexList(new ArrayList()); + } + @SuppressWarnings("unchecked") + public List getRegexList() { + return (List) kp.get("regexList"); + } + public void setRegexList(List patterns) { + kp.put("regexList", patterns); + } + + /** + * True if the list of regular expression should be considered as logically + * AND when matching. False if the list of regular expressions should be + * considered as logically OR when matching. + */ + { + setListLogicalOr(true); + } + public boolean getListLogicalOr() { + return (Boolean) kp.get("listLogicalOr"); + } + public void setListLogicalOr(boolean listLogicalOr) { + kp.put("listLogicalOr",listLogicalOr); + } + + /** + * Usual constructor. + */ + public MatchesListRegexDecideRule() { + } + + /** + * Evaluate whether given object's string version + * matches configured regexes + */ + @Override + protected boolean evaluate(CrawlURI uri) { + List regexes = getRegexList(); + if(regexes.size()==0){ + return false; + } + + String str = uri.toString(); + boolean listLogicOR = getListLogicalOr(); + + for (Pattern p: regexes) { + boolean matches = p.matcher(str).matches(); + + if (logger.isLoggable(Level.FINER)) { + logger.finer("Tested '" + str + "' match with regex '" + + p.pattern() + " and result was " + matches); + } + + if(matches){ + if(listLogicOR){ + // OR based and we just got a match, done! + logger.fine("Matched: " + str); + return true; + } + } else { + if(listLogicOR == false){ + // AND based and we just found a non-match, done! + return false; + } + } + } + + if (listLogicOR) { + return false; + } else { + return true; + } + } + +} \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/deciderules/MatchesRegexDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/MatchesRegexDecideRule.java new file mode 100644 index 00000000..2c7eeacc --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/MatchesRegexDecideRule.java @@ -0,0 +1,68 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.deciderules; + +import java.util.regex.Pattern; + +import org.archive.modules.CrawlURI; + +/** + * Rule applies configured decision to any CrawlURIs whose String URI + * matches the supplied regex. + * + * @author gojomo + */ +public class MatchesRegexDecideRule extends PredicatedDecideRule { + + private static final long serialVersionUID = 2L; + + { + setRegex(Pattern.compile(".")); + } + public Pattern getRegex() { + return (Pattern) kp.get("regex"); + } + public void setRegex(Pattern regex) { + kp.put("regex",regex); + } + + /** + * Usual constructor. + */ + public MatchesRegexDecideRule() { + } + + + /** + * Evaluate whether given object's string version + * matches configured regex + * + * @param object + * @return true if regex is matched + */ + @Override + protected boolean evaluate(CrawlURI uri) { + Pattern p = getRegex(); + return p.matcher(getString(uri)).matches(); + } + + protected String getString(CrawlURI uri) { + return uri.toString(); + } +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/NotMatchesListRegexDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/NotMatchesListRegexDecideRule.java new file mode 100644 index 00000000..9c0b5363 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/NotMatchesListRegexDecideRule.java @@ -0,0 +1,50 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.deciderules; + +import org.archive.modules.CrawlURI; + + +/** + * Rule applies configured decision to any URIs which do *not* + * match the supplied regex. + * + * @author Kristinn Sigurdsson + */ +public class NotMatchesListRegexDecideRule extends MatchesListRegexDecideRule { + private static final long serialVersionUID = 8691360087063555583L; + + /** + * Usual constructor. + */ + public NotMatchesListRegexDecideRule() { + } + + /** + * Evaluate whether given object's string version does not match + * configured regexs (by reversing the superclass's answer). + * + * @param object Object to make decision about. + * @return true if the regexs are not matched + */ + @Override + protected boolean evaluate(CrawlURI object) { + return ! super.evaluate(object); + } +} diff --git a/modules/src/main/java/org/archive/modules/deciderules/NotMatchesRegexDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/NotMatchesRegexDecideRule.java new file mode 100644 index 00000000..04f41b4e --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/NotMatchesRegexDecideRule.java @@ -0,0 +1,50 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.deciderules; + +import org.archive.modules.CrawlURI; + + +/** + * Rule applies configured decision to any URIs which do *not* + * match the supplied regex. + * + * @author gojomo + */ +public class NotMatchesRegexDecideRule extends MatchesRegexDecideRule { + private static final long serialVersionUID = -2085313401991694306L; + + /** + * Usual constructor. + */ + public NotMatchesRegexDecideRule(String name) { + } + + /** + * Evaluate whether given object's string version does not match + * configured regex (by reversing the superclass's answer). + * + * @param object Object to make decision about. + * @return true if the regex is not matched + */ + @Override + protected boolean evaluate(CrawlURI object) { + return ! super.evaluate(object); + } +} \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/seeds/SeedFileIterator.java b/modules/src/main/java/org/archive/modules/seeds/SeedFileIterator.java index 02eb6307..da717517 100644 --- a/modules/src/main/java/org/archive/modules/seeds/SeedFileIterator.java +++ b/modules/src/main/java/org/archive/modules/seeds/SeedFileIterator.java @@ -34,7 +34,7 @@ import org.apache.commons.httpclient.URIException; import org.archive.net.UURI; import org.archive.net.UURIFactory; import org.archive.util.iterator.LineReadingIterator; -import org.archive.util.iterator.RegexpLineIterator; +import org.archive.util.iterator.RegexLineIterator; import org.archive.util.iterator.TransformingIteratorWrapper; @@ -70,11 +70,11 @@ public class SeedFileIterator extends TransformingIteratorWrapper { */ public SeedFileIterator(BufferedReader inputReader, Writer ignoredWriter) { super(); - inner = new RegexpLineIterator( + inner = new RegexLineIterator( new LineReadingIterator(inputReader), - RegexpLineIterator.COMMENT_LINE, - RegexpLineIterator.NONWHITESPACE_ENTRY_TRAILING_COMMENT, - RegexpLineIterator.ENTRY); + RegexLineIterator.COMMENT_LINE, + RegexLineIterator.NONWHITESPACE_ENTRY_TRAILING_COMMENT, + RegexLineIterator.ENTRY); input = inputReader; ignored = ignoredWriter; } diff --git a/modules/src/test/java/org/archive/modules/deciderules/HasViaDecideRuleTest.java b/modules/src/test/java/org/archive/modules/deciderules/HasViaDecideRuleTest.java index 72a30b8c..d59e54bf 100644 --- a/modules/src/test/java/org/archive/modules/deciderules/HasViaDecideRuleTest.java +++ b/modules/src/test/java/org/archive/modules/deciderules/HasViaDecideRuleTest.java @@ -1,25 +1,20 @@ -/* Copyright (C) 2008 Internet Archive. +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * This file is part of the Heritrix web crawler (crawler.archive.org). + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. + * http://www.apache.org/licenses/LICENSE-2.0 * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * HopsPathMatchesRegExpDecideRuleTest.java - * Created on January 25, 2008 - * - * $Header$ + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.modules.deciderules; diff --git a/modules/src/test/java/org/archive/modules/deciderules/HopsPathMatchesRegExpDecideRuleTest.java b/modules/src/test/java/org/archive/modules/deciderules/HopsPathMatchesRegExpDecideRuleTest.java deleted file mode 100644 index faaea6b8..00000000 --- a/modules/src/test/java/org/archive/modules/deciderules/HopsPathMatchesRegExpDecideRuleTest.java +++ /dev/null @@ -1,37 +0,0 @@ -/* Copyright (C) 2008 Internet Archive. - * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * HopsPathMatchesRegExpDecideRuleTest.java - * Created on January 25, 2008 - * - * $Header$ - */ -package org.archive.modules.deciderules; - -import org.archive.state.ModuleTestBase; - -/** - * Generic Module unit test - * - */ -public class HopsPathMatchesRegExpDecideRuleTest extends ModuleTestBase { - // NOTE: even if empty, existence triggers minimal key -registration - // and -documentation tests from superclass - - // TODO: add class-specific tests -} diff --git a/modules/src/test/java/org/archive/modules/deciderules/HopsPathMatchesRegexDecideRuleTest.java b/modules/src/test/java/org/archive/modules/deciderules/HopsPathMatchesRegexDecideRuleTest.java new file mode 100644 index 00000000..781d5220 --- /dev/null +++ b/modules/src/test/java/org/archive/modules/deciderules/HopsPathMatchesRegexDecideRuleTest.java @@ -0,0 +1,32 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.deciderules; + +import org.archive.state.ModuleTestBase; + +/** + * Generic Module unit test + * + */ +public class HopsPathMatchesRegexDecideRuleTest extends ModuleTestBase { + // NOTE: even if empty, existence triggers minimal key -registration + // and -documentation tests from superclass + + // TODO: add class-specific tests +} diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java index b46523f5..67184e38 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java @@ -27,7 +27,9 @@ import java.util.List; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.Predicate; import org.apache.commons.httpclient.URIException; +import org.archive.modules.CrawlMetadata; import org.archive.modules.CrawlURI; +import org.archive.modules.net.RobotsHonoringPolicy; import org.archive.net.UURI; import org.archive.net.UURIFactory; import org.archive.util.Recorder; @@ -76,8 +78,11 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { @Override protected Extractor makeExtractor() { ExtractorHTML result = new ExtractorHTML(); - UriErrorLoggerModule ulm = new UnitTestUriLoggerModule(); + UriErrorLoggerModule ulm = new UnitTestUriLoggerModule(); result.setLoggerModule(ulm); + CrawlMetadata metadata = new CrawlMetadata(); + metadata.setRobotsHonoringPolicy(new RobotsHonoringPolicy()); + result.setMetadata(metadata); result.afterPropertiesSet(); return result; } diff --git a/modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java b/modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java index 6c3bf03a..f2ef95d3 100644 --- a/modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java @@ -23,7 +23,9 @@ package org.archive.modules.extractor; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.Predicate; import org.apache.commons.httpclient.URIException; +import org.archive.modules.CrawlMetadata; import org.archive.modules.CrawlURI; +import org.archive.modules.net.RobotsHonoringPolicy; import org.archive.net.UURI; import org.archive.net.UURIFactory; @@ -41,6 +43,9 @@ public class JerichoExtractorHTMLTest extends ExtractorHTMLTest { JerichoExtractorHTML result = new JerichoExtractorHTML(); UriErrorLoggerModule ulm = new UnitTestUriLoggerModule(); result.setLoggerModule(ulm); + CrawlMetadata metadata = new CrawlMetadata(); + metadata.setRobotsHonoringPolicy(new RobotsHonoringPolicy()); + result.setMetadata(metadata); result.afterPropertiesSet(); return result; }