Added pika support to the bin/run_eridu.py script.

Author: bluepizzey

Diff for: bin/run_eridu.py

@@ -1,12 +1,15 @@
 # -*- coding: utf-8 -*-
 
+import json
 import time
 
 import click
 import schedule
+import pika
 
 from eridu.logger import logger
 from eridu.config import FILTER_TAGS, SECONDS_BETWEEN_REQUESTS
+from eridu.config import QUEUE_HOST, QUEUE_PORT, EXCHANGE, QUESTIONS_ROUTING_KEY, ANSWERS_ROUTING_KEY
 from eridu.core import get_post_ids, split_post_ids, get_questions, get_answers, filter_posts_by_tag
 
 @click.command()
@@ -27,26 +30,39 @@ def run(params):
         page = params.get('page')
         logger.info('Getting posts for page {}.'.format(page))
 
+        connection = pika.BlockingConnection(pika.ConnectionParameters(
+            host=QUEUE_HOST,
+            port=QUEUE_PORT)
+        )
+        channel = connection.channel()
+
+        channel.exchange_declare(exchange=EXCHANGE,
+                                 type='topic')
+
         post_ids = get_post_ids(page)
         ids = split_post_ids(post_ids['items'])
 
         questions = get_questions(ids['question_ids'])
         questions = filter_posts_by_tag(questions['items'], tags)
-        for question in questions:
-            try:
-                print(question)
-            except UnicodeEncodeError:
-                pass
 
-        print('\n\n')
+        routing_key = QUESTIONS_ROUTING_KEY
+        for question in questions:
+            message = json.dumps(question)
+            channel.basic_publish(exchange=EXCHANGE,
+                                  routing_key=routing_key,
+                                  body=message)
 
         answers = get_answers(ids['answer_ids'])
         answers = filter_posts_by_tag(answers['items'], tags)
+
+        routing_key = ANSWERS_ROUTING_KEY
         for answer in answers:
-            try:
-                print(answer)
-            except UnicodeEncodeError:
-                pass
+            message = json.dumps(answer)
+            channel.basic_publish(exchange=EXCHANGE,
+                                  routing_key=routing_key,
+                                  body=message)
+
+        connection.close()
 
         params['page'] += 1
 

Diff for: eridu/config.py

@@ -22,3 +22,9 @@
 
 FILTER_TAGS = os.environ.get('ERIDU_FILTER_TAGS', 'python,python-2.x,python-3.x')
 SECONDS_BETWEEN_REQUESTS = int(os.environ.get('ERIDU_SECONDS_BETWEEN_REQUESTS', 300))
+
+QUEUE_HOST = os.environ.get('ERIDU_RABBITMQ_HOST', 'docker.local')
+QUEUE_PORT = int(os.environ.get('ERIDU_RABBITMQ_PORT', 32769))
+EXCHANGE = os.environ.get('ERIDU_RABBITMQ_EXCHANGE', 'eridu')
+QUESTIONS_ROUTING_KEY = os.environ.get('ERIDU_RABBITMQ_QUESTIONS_ROUTING_KEY', 'posts.historical.questions')
+ANSWERS_ROUTING_KEY = os.environ.get('ERIDU_RABBITMQ_ANSWERS_ROUTING_KEY', 'posts.historical.answers')

Diff for: requirements.txt

@@ -1,3 +1,4 @@
 Click>=6.0
 schedule==0.3.2
 requests==2.10.0
+pika==0.10.0