WARCSpout: add metadata field "fetch.statusCode" (holds HTTP status c…

…ode) (apache#823)
sebastian-nagel · Sep 24, 2020 · 3dcacfa · 3dcacfa
1 parent 1c4f08b
commit 3dcacfa
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/WARCSpout.java b/external/warc/src/main/java/com/digitalpebble/stormcrawler/warc/WARCSpout.java
@@ -434,6 +434,9 @@ public void nextTuple() {
 
         Metadata metadata = new Metadata();
 
+        // add HTTP status code expected by schedulers
+        metadata.addValue("fetch.statusCode", Integer.toString(http.status()));
+
         // Add HTTP response headers to metadata
         for (Map.Entry<String, List<String>> e : http.headers().map()
                 .entrySet()) {