Only copy source tag if not null.

internetarchive · Apr 20, 2021 · 52093eb · 52093eb
1 parent 308cee8
commit 52093eb
Showing 1 changed file with 4 additions and 5 deletions.
diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorRobotsTxt.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorRobotsTxt.java
@@ -12,9 +12,6 @@
 
 import org.apache.commons.httpclient.URIException;
 import org.archive.modules.CrawlURI;
-import org.archive.modules.extractor.ContentExtractor;
-import org.archive.modules.extractor.Hop;
-import org.archive.modules.extractor.LinkContext;
 
 public class ExtractorRobotsTxt extends ContentExtractor {
     private static final Logger LOGGER = Logger
@@ -65,8 +62,10 @@ protected boolean innerExtract(CrawlURI curi) {
                     curi.getViaContext());
 
             // Also copy the source over:
-            curiClone.setSourceTag(curi.getSourceTag());
-
+            if (curi.getSourceTag() != null) {
+                curiClone.setSourceTag(curi.getSourceTag());
+            }
+
             // Parse the robots for the sitemaps.
             List<String> links = parseRobotsTxt(
                     curi.getRecorder()