mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-09 07:24:41 +00:00
[HER-1480] URIs logged (as -6 or -7) that should never be scheduled
* LaxURI.java
increase tolerance with regard to path-segments
* UURIFactory.java
allow digits in scheme regex
discard old supported-schemes code
* UURIFactoryTest.java
tests for roundtrip serialization of problematic hostlike-scheme and bars-in-path
* CrawlerLoggerModule.java
discard special-casing of URI scheme errors
* SchemeNotInSetDecideRule.java
rule to apply to URIs of unknown schemes; by default REJECTs those not usually handled by Heritrix
* **/profile-crawler-beans.cxml
add REJECT SchemeNotInSetDecideRule
This commit is contained in:
1 parent
0347269785
commit
bc698c09fc
7 files changed
+118
-63
No files matched your search
@@ -43,7 +43,6 @@ import org.archive.crawler.util.Logs;
|
||||
import org.archive.io.GenerationFileHandler;
|
||||
import org.archive.modules.extractor.UriErrorLoggerModule;
|
||||
import org.archive.net.UURI;
|
||||
import org.archive.net.UURIFactory;
|
||||
import org.archive.spring.ConfigPath;
|
||||
import org.archive.util.ArchiveUtils;
|
||||
import org.springframework.beans.factory.InitializingBean;
|
||||
@@ -418,10 +417,6 @@ public class CrawlerLoggerModule
|
||||
* @param l String which could not be interpreted as URI without exception
|
||||
*/
|
||||
public void logUriError(URIException e, UURI u, CharSequence l) {
|
||||
if (e.getReasonCode() == UURIFactory.IGNORED_SCHEME) {
|
||||
// don't log those that are intentionally ignored
|
||||
return;
|
||||
}
|
||||
Object[] array = {u, l};
|
||||
uriErrors.log(Level.INFO, e.getMessage(), array);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user