Merge pull request #40 from topcoder-platform/develop

replace kafka simple consumer with group consumer

.circleci/config.yml

@@ -7,6 +7,7 @@ install_dependency: &install_dependency
           command: |
             sudo apt install jq
             sudo pip install awscli --upgrade
+            sudo apt install python-dev libffi-dev gcc libc-dev make
             sudo pip install docker-compose
 
 install_deploysuite: &install_deploysuite

README.md

@@ -20,6 +20,7 @@ The following parameters can be set in config files or in env variables:
 - KAFKA_CLIENT_CERT_KEY: Kafka connection private key, optional;
     if not provided, then SSL connection is not used, direct insecure connection is used;
     if provided, it can be either path to private key file or private key content
+- KAFKA_GROUP_ID: the Kafka group id, default value is 'submission-processor'
 - SUBMISSION_CREATE_TOPIC: Kafka topic related to Submission creation, default value is 'submission.notification.create'
 - AVSCAN_TOPIC: Kafka topic related to AV Scan, default value is 'avscan.action.scan'
 - ACCESS_KEY_ID: the AWS access key id

config/default.js

@@ -6,6 +6,9 @@ module.exports = {
   LOG_LEVEL: process.env.LOG_LEVEL || 'debug',
   MAXFILESIZE: process.env.MAXFILESIZE || 4294967296,
 
+  // Kafka group id
+  KAFKA_GROUP_ID: process.env.KAFKA_GROUP_ID || 'submission-processor',
+
   KAFKA_URL: process.env.KAFKA_URL || 'localhost:9092',
   KAFKA_CLIENT_CERT: process.env.KAFKA_CLIENT_CERT,
   KAFKA_CLIENT_CERT_KEY: process.env.KAFKA_CLIENT_CERT_KEY,