diff --git a/modules/src/main/java/org/archive/modules/deciderules/ContentTypeMatchesRegExpDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/ContentTypeMatchesRegExpDecideRule.java deleted file mode 100644 index 8f910800..00000000 --- a/modules/src/main/java/org/archive/modules/deciderules/ContentTypeMatchesRegExpDecideRule.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.modules.deciderules; - -import org.archive.modules.CrawlURI; - - -/** - * DecideRule whose decision is applied if the URI's content-type - * is present and matches the supplied regular expression. - * - * @author Olaf Freyer - */ -public class ContentTypeMatchesRegExpDecideRule extends MatchesRegExpDecideRule{ - private static final long serialVersionUID = -2066930281015155843L; - - public ContentTypeMatchesRegExpDecideRule() { - } - - - @Override - protected String getString(CrawlURI uri) { - return uri.getContentType(); - } -} diff --git a/modules/src/main/java/org/archive/modules/deciderules/ContentTypeNotMatchesRegExpDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/ContentTypeNotMatchesRegExpDecideRule.java deleted file mode 100644 index 039ecace..00000000 --- a/modules/src/main/java/org/archive/modules/deciderules/ContentTypeNotMatchesRegExpDecideRule.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.modules.deciderules; - -import org.archive.modules.CrawlURI; - -/** - * DecideRule whose decision is applied if the URI's content-type - * is present and does not match the supplied regular expression. - * - * @author Olaf Freyer - */ -public class ContentTypeNotMatchesRegExpDecideRule extends - ContentTypeMatchesRegExpDecideRule { - private static final long serialVersionUID = 4729800377757426137L; - - public ContentTypeNotMatchesRegExpDecideRule() { - } - - /** - * Evaluate whether given object's string version does not match - * configured regexp (by reversing the superclass's answer). - * - * @param object Object to make decision about. - * @return true if the regexp is not matched - */ - @Override - protected boolean evaluate(CrawlURI o) { - return !super.evaluate(o); - } - -} diff --git a/modules/src/main/java/org/archive/modules/deciderules/FetchStatusMatchesRegExpDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/FetchStatusMatchesRegExpDecideRule.java deleted file mode 100644 index 2e96044a..00000000 --- a/modules/src/main/java/org/archive/modules/deciderules/FetchStatusMatchesRegExpDecideRule.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.modules.deciderules; - - -import org.archive.modules.CrawlURI; - - - -public class FetchStatusMatchesRegExpDecideRule extends MatchesRegExpDecideRule { - - private static final long serialVersionUID = 3L; - - - /** - * Usual constructor. - */ - public FetchStatusMatchesRegExpDecideRule() { - } - - - protected String getString(CrawlURI uri) { - return Integer.toString(uri.getFetchStatus()); - } -} \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/deciderules/FetchStatusNotMatchesRegExpDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/FetchStatusNotMatchesRegExpDecideRule.java deleted file mode 100644 index 518d843d..00000000 --- a/modules/src/main/java/org/archive/modules/deciderules/FetchStatusNotMatchesRegExpDecideRule.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.modules.deciderules; - -import org.archive.modules.CrawlURI; - - -public class FetchStatusNotMatchesRegExpDecideRule -extends FetchStatusMatchesRegExpDecideRule { - - private static final long serialVersionUID = -2220182698344063577L; -// private final Logger logger = Logger.getLogger(this.getClass().getName()); - - /** - * Usual constructor. - * @param name - */ - public FetchStatusNotMatchesRegExpDecideRule() { - } - - /** - * Evaluate whether given object's FetchStatus does not match - * configured regexp (by reversing the superclass's answer). - * - * @param object Object to make decision about. - * @return true if the regexp is not matched - */ - @Override - protected boolean evaluate(CrawlURI object) { - return ! super.evaluate(object); - } -} \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/deciderules/HopsPathMatchesRegExpDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/HopsPathMatchesRegExpDecideRule.java deleted file mode 100644 index 3a0945b1..00000000 --- a/modules/src/main/java/org/archive/modules/deciderules/HopsPathMatchesRegExpDecideRule.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.modules.deciderules; - -import org.archive.modules.CrawlURI; - -/** - * Rule applies configured decision to any CrawlURIs whose 'hops-path' - * (string like "LLXE" etc.) matches the supplied regexp. - * - * @author gojomo - */ -public class HopsPathMatchesRegExpDecideRule extends MatchesRegExpDecideRule { - private static final long serialVersionUID = 3L; - - /** - * Usual constructor. - * @param name - */ - public HopsPathMatchesRegExpDecideRule() { - } - - - @Override - protected String getString(CrawlURI uri) { - return uri.getPathFromSeed(); - } -} diff --git a/modules/src/main/java/org/archive/modules/deciderules/MatchesListRegExpDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/MatchesListRegExpDecideRule.java deleted file mode 100644 index 92a1690e..00000000 --- a/modules/src/main/java/org/archive/modules/deciderules/MatchesListRegExpDecideRule.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.modules.deciderules; - -import java.util.ArrayList; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.regex.Pattern; - -import org.archive.modules.CrawlURI; - -/** - * Rule applies configured decision to any CrawlURIs whose String URI - * matches the supplied regexps. - *

- * The list of regular expressions can be considered logically AND or OR. - * - * @author Kristinn Sigurdsson - * - * @see MatchesRegExpDecideRule - */ -public class MatchesListRegExpDecideRule extends PredicatedDecideRule { - - - private static final long serialVersionUID = 3L; - - private static final Logger logger = - Logger.getLogger(MatchesListRegExpDecideRule.class.getName()); - - - /** - * The list of regular expressions to evalute against the URI. - */ - { - setRegexList(new ArrayList()); - } - @SuppressWarnings("unchecked") - public List getRegexList() { - return (List) kp.get("regexList"); - } - public void setRegexList(List patterns) { - kp.put("regexList", patterns); - } - - /** - * True if the list of regular expression should be considered as logically - * AND when matching. False if the list of regular expressions should be - * considered as logically OR when matching. - */ - { - setListLogicalOr(true); - } - public boolean getListLogicalOr() { - return (Boolean) kp.get("listLogicalOr"); - } - public void setListLogicalOr(boolean listLogicalOr) { - kp.put("listLogicalOr",listLogicalOr); - } - - /** - * Usual constructor. - */ - public MatchesListRegExpDecideRule() { - } - - /** - * Evaluate whether given object's string version - * matches configured regexps - */ - @Override - protected boolean evaluate(CrawlURI uri) { - List regexps = getRegexList(); - if(regexps.size()==0){ - return false; - } - - String str = uri.toString(); - boolean listLogicOR = getListLogicalOr(); - - for (Pattern p: regexps) { - boolean matches = p.matcher(str).matches(); - - if (logger.isLoggable(Level.FINER)) { - logger.finer("Tested '" + str + "' match with regex '" + - p.pattern() + " and result was " + matches); - } - - if(matches){ - if(listLogicOR){ - // OR based and we just got a match, done! - logger.fine("Matched: " + str); - return true; - } - } else { - if(listLogicOR == false){ - // AND based and we just found a non-match, done! - return false; - } - } - } - - if (listLogicOR) { - return false; - } else { - return true; - } - } - -} \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/deciderules/MatchesRegExpDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/MatchesRegExpDecideRule.java deleted file mode 100644 index 1f8537ab..00000000 --- a/modules/src/main/java/org/archive/modules/deciderules/MatchesRegExpDecideRule.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.modules.deciderules; - -import java.util.regex.Pattern; - -import org.archive.modules.CrawlURI; - -/** - * Rule applies configured decision to any CrawlURIs whose String URI - * matches the supplied regexp. - * - * @author gojomo - */ -public class MatchesRegExpDecideRule extends PredicatedDecideRule { - - private static final long serialVersionUID = 2L; - - { - setRegex(Pattern.compile(".")); - } - public Pattern getRegex() { - return (Pattern) kp.get("regex"); - } - public void setRegex(Pattern regex) { - kp.put("regex",regex); - } - - /** - * Usual constructor. - */ - public MatchesRegExpDecideRule() { - } - - - /** - * Evaluate whether given object's string version - * matches configured regexp - * - * @param object - * @return true if regexp is matched - */ - @Override - protected boolean evaluate(CrawlURI uri) { - Pattern p = getRegex(); - return p.matcher(getString(uri)).matches(); - } - - protected String getString(CrawlURI uri) { - return uri.toString(); - } -} diff --git a/modules/src/main/java/org/archive/modules/deciderules/NotMatchesFilePatternDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/NotMatchesFilePatternDecideRule.java index 89a38390..22498358 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/NotMatchesFilePatternDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/NotMatchesFilePatternDecideRule.java @@ -23,7 +23,7 @@ import org.archive.modules.CrawlURI; /** * Rule applies configured decision to any URIs which do *not* - * match the supplied (file-pattern) regexp. + * match the supplied (file-pattern) regex. * * @author gojomo */ @@ -32,7 +32,7 @@ extends MatchesFilePatternDecideRule { private static final long serialVersionUID = -8161371026787859554L; //private static final Logger logger = - // Logger.getLogger(NotMatchesRegExpDecideRule.class.getName()); + // Logger.getLogger(NotMatchesRegexDecideRule.class.getName()); /** * Usual constructor. @@ -43,10 +43,10 @@ extends MatchesFilePatternDecideRule { /** * Evaluate whether given object's string version does not match - * configured regexp (by reversing the superclass's answer). + * configured regex (by reversing the superclass's answer). * * @param object Object to make decision about. - * @return true if the regexp is not matched + * @return true if the regex is not matched */ @Override protected boolean evaluate(CrawlURI uri) { diff --git a/modules/src/main/java/org/archive/modules/deciderules/NotMatchesListRegExpDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/NotMatchesListRegExpDecideRule.java deleted file mode 100644 index d9d4633c..00000000 --- a/modules/src/main/java/org/archive/modules/deciderules/NotMatchesListRegExpDecideRule.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.modules.deciderules; - -import org.archive.modules.CrawlURI; - - -/** - * Rule applies configured decision to any URIs which do *not* - * match the supplied regexp. - * - * @author Kristinn Sigurdsson - */ -public class NotMatchesListRegExpDecideRule extends MatchesListRegExpDecideRule { - - private static final long serialVersionUID = 8691360087063555583L; - - //private static final Logger logger = - // Logger.getLogger(NotMatchesListRegExpDecideRule.class.getName()); - - - /** - * Usual constructor. - * @param name - */ - public NotMatchesListRegExpDecideRule() { - } - - /** - * Evaluate whether given object's string version does not match - * configured regexps (by reversing the superclass's answer). - * - * @param object Object to make decision about. - * @return true if the regexps are not matched - */ - @Override - protected boolean evaluate(CrawlURI object) { - return ! super.evaluate(object); - } -} diff --git a/modules/src/main/java/org/archive/modules/deciderules/NotMatchesRegExpDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/NotMatchesRegExpDecideRule.java deleted file mode 100644 index 9b8e1252..00000000 --- a/modules/src/main/java/org/archive/modules/deciderules/NotMatchesRegExpDecideRule.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.modules.deciderules; - -import org.archive.modules.CrawlURI; - - -/** - * Rule applies configured decision to any URIs which do *not* - * match the supplied regexp. - * - * @author gojomo - */ -public class NotMatchesRegExpDecideRule extends MatchesRegExpDecideRule { - - private static final long serialVersionUID = -2085313401991694306L; - //private static final Logger logger = - // Logger.getLogger(NotMatchesRegExpDecideRule.class.getName()); - - - /** - * Usual constructor. - */ - public NotMatchesRegExpDecideRule(String name) { - } - - /** - * Evaluate whether given object's string version does not match - * configured regexp (by reversing the superclass's answer). - * - * @param object Object to make decision about. - * @return true if the regexp is not matched - */ - @Override - protected boolean evaluate(CrawlURI object) { - return ! super.evaluate(object); - } -} diff --git a/modules/src/main/java/org/archive/modules/deciderules/PathologicalPathDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/PathologicalPathDecideRule.java index 81e70b04..4da1557d 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/PathologicalPathDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/PathologicalPathDecideRule.java @@ -32,7 +32,6 @@ import org.archive.modules.CrawlURI; * @author gojomo */ public class PathologicalPathDecideRule extends DecideRule { - private static final long serialVersionUID = 3L; /** @@ -72,9 +71,9 @@ public class PathologicalPathDecideRule extends DecideRule { } /** - * Construct the regexp string to be matched against the URI. + * Construct the regex string to be matched against the URI. * @param o an object to extract a URI from. - * @return the regexp pattern. + * @return the regex pattern. */ private Pattern getPattern(int maxRep) { // race no concern: assignment is atomic, happy with any last value @@ -82,13 +81,13 @@ public class PathologicalPathDecideRule extends DecideRule { if (p != null) { return p; } - String regex = constructRegexp(maxRep); + String regex = constructRegex(maxRep); p = Pattern.compile(regex); pattern.set(p); return p; } - protected String constructRegexp(int rep) { + protected String constructRegex(int rep) { return (rep == 0) ? null : ".*?/(.*?/)\\1{" + rep + ",}.*"; } } diff --git a/modules/src/main/java/org/archive/modules/deciderules/surt/NotSurtPrefixedDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/surt/NotSurtPrefixedDecideRule.java index 17e24ae0..83fcdaac 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/surt/NotSurtPrefixedDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/surt/NotSurtPrefixedDecideRule.java @@ -49,7 +49,7 @@ public class NotSurtPrefixedDecideRule extends SurtPrefixedDecideRule { * prefix set -- simply reverse superclass's determination * * @param object - * @return true if regexp is matched + * @return true if regex is matched */ protected boolean evaluate(CrawlURI object) { return !super.evaluate(object); diff --git a/modules/src/main/java/org/archive/modules/extractor/AggressiveExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/AggressiveExtractorHTML.java index 5e5cc4f9..6f8c1dd7 100644 --- a/modules/src/main/java/org/archive/modules/extractor/AggressiveExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/AggressiveExtractorHTML.java @@ -25,7 +25,7 @@ import org.archive.modules.CrawlURI; /** * Extended version of ExtractorHTML with more aggressive javascript link * extraction where javascript code is parsed first with general HTML tags - * regexp, and than by javascript speculative link regexp. + * regex, and than by javascript speculative link regex. * * @author Igor Ranitovic * @@ -57,7 +57,7 @@ extends ExtractorHTML { ret.append(super.report()); ret.append(" Function: Link extraction on HTML documents " + "(including embedded CSS)\n"); - ret.append(" ProcessorURRIs handled: " + numberOfCURIsHandled + "\n"); + ret.append(" ProcessorURIs handled: " + numberOfCURIsHandled + "\n"); ret.append(" Links extracted: " + numberOfLinksExtracted + "\n"); return ret.toString(); } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index b2e17384..7d0c22e3 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -29,6 +29,7 @@ import java.util.regex.Pattern; import org.apache.commons.httpclient.URIException; import org.archive.io.ReplayCharSequence; +import org.archive.modules.CrawlMetadata; import org.archive.modules.CrawlURI; import org.archive.modules.net.RobotsHonoringPolicy; import org.archive.net.UURI; @@ -287,26 +288,24 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean } /** - * The robots honoring policy to use when considering a robots META tag. + * CrawlMetadata provides the robots honoring policy to use when + * considering a robots META tag. */ - public RobotsHonoringPolicy getRobotsHonoringPolicy() { - return (RobotsHonoringPolicy) kp.get("robotsHonoringPolicy"); + CrawlMetadata metadata; + public CrawlMetadata getMetadata() { + return metadata; } @Autowired - public void setRobotsHonoringPolicy(RobotsHonoringPolicy policy) { - kp.put("robotsHonoringPolicy",policy); + public void setMetadata(CrawlMetadata provider) { + this.metadata = provider; } protected long numberOfCURIsHandled = 0; protected long numberOfLinksExtracted = 0; - - - RobotsHonoringPolicy honoringPolicy; private Pattern relevantTagExtractor; private Pattern eachAttributeExtractor; - - + public ExtractorHTML() { } @@ -765,7 +764,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean // Look for the 'robots' meta-tag if("robots".equalsIgnoreCase(name) && content != null ) { curi.getData().put(A_META_ROBOTS, content); - RobotsHonoringPolicy policy = honoringPolicy; + RobotsHonoringPolicy policy = metadata.getRobotsHonoringPolicy(); String contentLower = content.toLowerCase(); if ((policy == null || (!policy.isType(RobotsHonoringPolicy.Type.IGNORE) diff --git a/modules/src/main/java/org/archive/modules/extractor/HTTPContentDigest.java b/modules/src/main/java/org/archive/modules/extractor/HTTPContentDigest.java index 2d7299f2..81202b04 100644 --- a/modules/src/main/java/org/archive/modules/extractor/HTTPContentDigest.java +++ b/modules/src/main/java/org/archive/modules/extractor/HTTPContentDigest.java @@ -115,8 +115,8 @@ public class HTTPContentDigest extends Processor { protected void innerProcess(CrawlURI curi) throws InterruptedException { // Ok, if we got this far we need to calculate the content digest. - // Get the regexpr - Pattern regexpr = getStripRegex(); + // Get the regex + Pattern regex = getStripRegex(); // Get a replay of the document character seq. ReplayCharSequence cs = null; @@ -135,11 +135,11 @@ public class HTTPContentDigest extends Processor { String s = null; - if (regexpr != null) { + if (regex != null) { s = cs.toString(); } else { // Process the document - Matcher m = regexpr.matcher(cs); + Matcher m = regex.matcher(cs); s = m.replaceAll(" "); } digest.update(s.getBytes()); diff --git a/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java index dafed432..2c306277 100644 --- a/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java @@ -289,7 +289,7 @@ public class JerichoExtractorHTML extends ExtractorHTML { if ("robots".equals(name) && content != null) { curi.getData().put(A_META_ROBOTS, content); - RobotsHonoringPolicy policy = honoringPolicy; + RobotsHonoringPolicy policy = metadata.getRobotsHonoringPolicy(); String contentLower = content.toLowerCase(); if ((policy == null || (!policy.isType( RobotsHonoringPolicy.Type.IGNORE) && !policy.isType( diff --git a/modules/src/main/java/org/archive/modules/seeds/TextSeedModule.java b/modules/src/main/java/org/archive/modules/seeds/TextSeedModule.java index 18d94768..ab63697e 100644 --- a/modules/src/main/java/org/archive/modules/seeds/TextSeedModule.java +++ b/modules/src/main/java/org/archive/modules/seeds/TextSeedModule.java @@ -44,7 +44,7 @@ import org.archive.net.UURIFactory; import org.archive.spring.WriteTarget; import org.archive.util.DevUtils; import org.archive.util.iterator.LineReadingIterator; -import org.archive.util.iterator.RegexpLineIterator; +import org.archive.util.iterator.RegexLineIterator; import org.springframework.beans.factory.annotation.Required; /** @@ -100,11 +100,11 @@ implements ReadSource, protected void announceSeedsFromReader(BufferedReader reader) { String s; Iterator iter = - new RegexpLineIterator( + new RegexLineIterator( new LineReadingIterator(reader), - RegexpLineIterator.COMMENT_LINE, - RegexpLineIterator.NONWHITESPACE_ENTRY_TRAILING_COMMENT, - RegexpLineIterator.ENTRY); + RegexLineIterator.COMMENT_LINE, + RegexLineIterator.NONWHITESPACE_ENTRY_TRAILING_COMMENT, + RegexLineIterator.ENTRY); while (iter.hasNext()) { s = (String) iter.next();