Merge pull request #341 from internetarchive/noplaylist-ydl

youtube-dl --no-playlist

Author: anjackson

contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java

@@ -44,6 +44,7 @@
 import java.util.logging.Logger;
 
 import org.apache.commons.httpclient.URIException;
+import org.archive.crawler.frontier.AMQPUrlReceiver;
 import org.archive.crawler.reporting.CrawlerLoggerModule;
 import org.archive.format.warc.WARCConstants.WARCRecordType;
 import org.archive.io.warc.WARCRecordInfo;
@@ -543,6 +544,11 @@ protected boolean shouldExtract(CrawlURI uri) {
             return false;
         }
 
+        // skip crawl uris received from umbra
+        if (uri.getAnnotations().contains(AMQPUrlReceiver.A_RECEIVED_FROM_AMQP)) {
+            return false;
+        }
+
         String mime = uri.getContentType().toLowerCase();
         if (mime.startsWith("text/html")
                 || mime.startsWith("application/xhtml")

modules/src/main/java/org/archive/modules/warc/FtpResponseRecordBuilder.java

@@ -32,7 +32,7 @@ public WARCRecordInfo buildRecord(CrawlURI curi, URI concurrentTo) throws IOExce
             recordInfo.addExtraHeader(HEADER_KEY_CONCURRENT_TO,
                     '<' + concurrentTo.toString() + '>');
         }
-        recordInfo.setType(WARCRecordType.resource);
+        recordInfo.setType(WARCRecordType.response);
         recordInfo.setUrl(curi.toString());
         recordInfo.setCreate14DigitDate(timestamp);
         recordInfo.setMimetype(curi.getContentType());