lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-22 21:55:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Only copy source tag if not null.

Browse Source
This commit is contained in:
Kristinn Sigurdsson
2021-04-20 13:18:29 +00:00
parent 308cee85a3
commit 52093eb25f
1 changed files with 4 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/extractor/ExtractorRobotsTxt.java
+4 -5
View File
@@ -12,9 +12,6 @@ import java.util.regex.Pattern;
import org.apache.commons.httpclient.URIException;
import org.archive.modules.CrawlURI;
import org.archive.modules.extractor.ContentExtractor;
import org.archive.modules.extractor.Hop;
import org.archive.modules.extractor.LinkContext;
public class ExtractorRobotsTxt extends ContentExtractor {
private static final Logger LOGGER = Logger
@@ -65,8 +62,10 @@ public class ExtractorRobotsTxt extends ContentExtractor {
curi.getViaContext());
// Also copy the source over:
curiClone.setSourceTag(curi.getSourceTag());
if (curi.getSourceTag() != null) {
curiClone.setSourceTag(curi.getSourceTag());
}
// Parse the robots for the sitemaps.
List<String> links = parseRobotsTxt(
curi.getRecorder()