File tree Expand file tree Collapse file tree 1 file changed +4
-5
lines changed
modules/src/main/java/org/archive/modules/extractor Expand file tree Collapse file tree 1 file changed +4
-5
lines changed Original file line number Diff line number Diff line change 12
12
13
13
import org .apache .commons .httpclient .URIException ;
14
14
import org .archive .modules .CrawlURI ;
15
- import org .archive .modules .extractor .ContentExtractor ;
16
- import org .archive .modules .extractor .Hop ;
17
- import org .archive .modules .extractor .LinkContext ;
18
15
19
16
public class ExtractorRobotsTxt extends ContentExtractor {
20
17
private static final Logger LOGGER = Logger
@@ -65,8 +62,10 @@ protected boolean innerExtract(CrawlURI curi) {
65
62
curi .getViaContext ());
66
63
67
64
// Also copy the source over:
68
- curiClone .setSourceTag (curi .getSourceTag ());
69
-
65
+ if (curi .getSourceTag () != null ) {
66
+ curiClone .setSourceTag (curi .getSourceTag ());
67
+ }
68
+
70
69
// Parse the robots for the sitemaps.
71
70
List <String > links = parseRobotsTxt (
72
71
curi .getRecorder ()
You can’t perform that action at this time.
0 commit comments