[HER-1480] URIs logged (as -6 or -7) that should never be scheduled

* LaxURI.java
    increase tolerance with regard to path-segments
* UURIFactory.java
    allow digits in scheme regex
    discard old supported-schemes code
* UURIFactoryTest.java
    tests for roundtrip serialization of problematic hostlike-scheme and bars-in-path
* CrawlerLoggerModule.java
    discard special-casing of URI scheme errors
* SchemeNotInSetDecideRule.java
    rule to apply to URIs of unknown schemes; by default REJECTs those not usually handled by Heritrix
* **/profile-crawler-beans.cxml
    add REJECT SchemeNotInSetDecideRule
This commit is contained in:
gojomo committed 2010-09-03 00:24:13 +00:00
1 parent 0347269785
commit bc698c09fc
7 files changed
+118 -63

No files matched your search

@@ -43,7 +43,6 @@ import org.archive.crawler.util.Logs;
import org.archive.io.GenerationFileHandler;
import org.archive.modules.extractor.UriErrorLoggerModule;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
import org.archive.spring.ConfigPath;
import org.archive.util.ArchiveUtils;
import org.springframework.beans.factory.InitializingBean;
@@ -418,10 +417,6 @@ public class CrawlerLoggerModule
* @param l String which could not be interpreted as URI without exception
*/
public void logUriError(URIException e, UURI u, CharSequence l) {
if (e.getReasonCode() == UURIFactory.IGNORED_SCHEME) {
// don't log those that are intentionally ignored
return;
}
Object[] array = {u, l};
uriErrors.log(Level.INFO, e.getMessage(), array);
}