adding PartitionerBolt again

clarin-eric · Apr 23, 2023 · 3d5971a · 3d5971a
1 parent 5727105
commit 3d5971a
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 6 deletions.
diff --git a/crawler-test.flux b/crawler-test.flux
@@ -15,6 +15,9 @@ spouts:
     parallelism: 1
 
 bolts:
+  - id: "partitioner"
+    className: "com.digitalpebble.stormcrawler.bolt.URLPartitionerBolt"
+    parallelism: 1
   - id: "fetcher"
     className: "eu.clarin.linkchecker.bolt.MetricsFetcherBolt"
     parallelism: 1
@@ -24,17 +27,22 @@ bolts:
 
 streams:
   - from: "spout"
-    to: "fetcher"
+    to: "partitioner"
     grouping:
       type: SHUFFLE
+  - from: "partitioner"
+    to: "fetcher"
+    grouping:
+      type: FIELDS
+      args: ["key"]
   - from: "fetcher"
     to: "status"
     grouping:
       type: FIELDS
       args: ["url"]
       streamId: "status"        
   - from: "fetcher"
-    to: "fetcher"
+    to: "partitioner"
     grouping:
       type: SHUFFLE
-      streamId: "redirect"
+      streamId: "redirect"
diff --git a/crawler.flux b/crawler.flux
@@ -15,6 +15,9 @@ spouts:
     parallelism: 1
 
 bolts:
+  - id: "partitioner"
+    className: "com.digitalpebble.stormcrawler.bolt.URLPartitionerBolt"
+    parallelism: 1
   - id: "fetcher"
     className: "eu.clarin.linkchecker.bolt.MetricsFetcherBolt"
     parallelism: 1
@@ -24,17 +27,22 @@ bolts:
 
 streams:
   - from: "spout"
-    to: "fetcher"
+    to: "partitioner"
     grouping:
       type: SHUFFLE
+  - from: "partitioner"
+    to: "fetcher"
+    grouping:
+      type: FIELDS
+      args: ["key"]
   - from: "fetcher"
     to: "status"
     grouping:
       type: FIELDS
       args: ["url"]
       streamId: "status"        
   - from: "fetcher"
-    to: "fetcher"
+    to: "partitioner"
     grouping:
       type: SHUFFLE
-      streamId: "redirect"
+      streamId: "redirect"