From 52093eb25f8f17b48eea5acad06bb6eaeb99adf4 Mon Sep 17 00:00:00 2001
From: Kristinn Sigurdsson <kristinn@landsbokasafn.is>
Date: Tue, 20 Apr 2021 13:18:29 +0000
Subject: [PATCH] Only copy source tag if not null.

---
 .../archive/modules/extractor/ExtractorRobotsTxt.java    | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorRobotsTxt.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorRobotsTxt.java
index 03aebca5..c63b23be 100644
--- a/modules/src/main/java/org/archive/modules/extractor/ExtractorRobotsTxt.java
+++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorRobotsTxt.java
@@ -12,9 +12,6 @@ import java.util.regex.Pattern;
 
 import org.apache.commons.httpclient.URIException;
 import org.archive.modules.CrawlURI;
-import org.archive.modules.extractor.ContentExtractor;
-import org.archive.modules.extractor.Hop;
-import org.archive.modules.extractor.LinkContext;
 
 public class ExtractorRobotsTxt extends ContentExtractor {
     private static final Logger LOGGER = Logger
@@ -65,8 +62,10 @@ public class ExtractorRobotsTxt extends ContentExtractor {
                     curi.getViaContext());
 
             // Also copy the source over:
-            curiClone.setSourceTag(curi.getSourceTag());
-
+            if (curi.getSourceTag() != null) {
+                curiClone.setSourceTag(curi.getSourceTag());
+            }
+            
             // Parse the robots for the sitemaps.
             List<String> links = parseRobotsTxt(
                     curi.getRecorder()