Add continuous integration on Heroku

* Remove the Rakefile, it caused build issues.
* Loosen the dependency on Active Support, bundler had problems.
* Add a consumer/producer pair.

Author: dasch

.gitignore

@@ -1,6 +1,5 @@
 /.bundle/
 /.yardoc
-/Gemfile.lock
 /_yardoc/
 /coverage/
 /doc/

Gemfile.lock

@@ -0,0 +1,79 @@
+PATH
+  remote: .
+  specs:
+    ruby-kafka (0.3.8)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.2.5)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    benchmark-perf (0.1.0)
+    builder (3.2.2)
+    coderay (1.1.0)
+    colored (1.2)
+    diff-lcs (1.2.5)
+    docker-api (1.25.0)
+      excon (>= 0.38.0)
+      json
+    dogstatsd-ruby (1.6.0)
+    dotenv (2.1.0)
+    excon (0.45.4)
+    i18n (0.7.0)
+    json (1.8.3)
+    method_source (0.8.2)
+    minitest (5.8.3)
+    pry (0.9.12.6)
+      coderay (~> 1.0)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    rake (10.5.0)
+    rspec (3.4.0)
+      rspec-core (~> 3.4.0)
+      rspec-expectations (~> 3.4.0)
+      rspec-mocks (~> 3.4.0)
+    rspec-benchmark (0.1.0)
+      benchmark-perf (~> 0.1.0)
+      rspec (>= 3.0.0, < 4.0.0)
+    rspec-core (3.4.1)
+      rspec-support (~> 3.4.0)
+    rspec-expectations (3.4.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.4.0)
+    rspec-mocks (3.4.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.4.0)
+    rspec-support (3.4.1)
+    rspec_junit_formatter (0.2.2)
+      builder (< 4)
+      rspec-core (>= 2, < 4, != 2.12.0)
+    slop (3.6.0)
+    snappy (0.0.12)
+    thread_safe (0.3.5)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport
+  bundler (>= 1.9.5)
+  colored
+  docker-api
+  dogstatsd-ruby
+  dotenv
+  pry
+  rake (~> 10.0)
+  rspec
+  rspec-benchmark
+  rspec_junit_formatter (= 0.2.2)
+  ruby-kafka!
+  snappy
+
+BUNDLED WITH
+   1.10.6

Procfile

@@ -0,0 +1,2 @@
+producer: bundle exec ruby ci/producer.rb
+consumer: bundle exec ruby ci/consumer.rb

Rakefile

@@ -0,0 +1,32 @@
+$LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
+
+require "kafka"
+
+logger = Logger.new(STDOUT)
+logger.level = Logger::INFO
+logger.formatter = -> (_, _, _, msg) { msg }
+
+STDOUT.sync = true
+
+kafka = Kafka.new(
+  logger: logger,
+  seed_brokers: ENV.fetch("HEROKU_KAFKA_URL"),
+  ssl_ca_cert: ENV.fetch("HEROKU_KAFKA_TRUSTED_CERT"),
+  ssl_client_cert: ENV.fetch("HEROKU_KAFKA_CLIENT_CERT"),
+  ssl_client_cert_key: ENV.fetch("HEROKU_KAFKA_CLIENT_CERT_KEY"),
+)
+
+consumer = kafka.consumer(group_id: "greetings-group")
+consumer.subscribe("greetings")
+
+num_messages = 0
+
+trap("TERM") { consumer.stop }
+
+consumer.each_message do |message|
+  num_messages += 1
+
+  if num_messages % 1000 == 0
+    puts "Processed #{num_messages} messages"
+  end
+end

ci/consumer.rb

@@ -0,0 +1,36 @@
+$LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
+
+require "kafka"
+
+logger = Logger.new(STDOUT)
+logger.level = Logger::INFO
+logger.formatter = -> (_, _, _, msg) { msg }
+
+STDOUT.sync = true
+
+kafka = Kafka.new(
+  logger: logger,
+  seed_brokers: ENV.fetch("HEROKU_KAFKA_URL"),
+  ssl_ca_cert: ENV.fetch("HEROKU_KAFKA_TRUSTED_CERT"),
+  ssl_client_cert: ENV.fetch("HEROKU_KAFKA_CLIENT_CERT"),
+  ssl_client_cert_key: ENV.fetch("HEROKU_KAFKA_CLIENT_CERT_KEY"),
+)
+
+producer = kafka.async_producer(
+  delivery_interval: 1,
+  max_queue_size: 5_000,
+  max_buffer_size: 10_000,
+)
+
+num_messages = 0
+
+trap("TERM") { producer.shutdown }
+
+loop do
+  begin
+    producer.produce("hello", key: "world", topic: "greetings")
+  rescue Kafka::BufferOverflow
+    puts "Buffer overflow, backing off..."
+    sleep 10
+  end
+end

ci/producer.rb

@@ -34,7 +34,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "dotenv"
   spec.add_development_dependency "docker-api"
   spec.add_development_dependency "rspec-benchmark"
-  spec.add_development_dependency "activesupport", ">= 4.2.0", "< 5.1"
+  spec.add_development_dependency "activesupport"
   spec.add_development_dependency "snappy"
   spec.add_development_dependency "colored"
   spec.add_development_dependency "rspec_junit_formatter", "0.2.2"