Add a middleware to kill all bots that don't respect robots.txt

Author: rafaelfranca

config/application.rb

@@ -9,6 +9,8 @@
 # you've limited to :test, :development, or :production.
 Bundler.require(*Rails.groups)
 
+require './lib/bot_killer'
+
 module RailsContributors
   class Application < Rails::Application
     # Initialize configuration defaults for originally generated Rails version.
@@ -17,5 +19,7 @@ class Application < Rails::Application
     # Settings in config/environments/* take precedence over those specified here.
     # Application configuration should go into files in config/initializers
     # -- all .rb files in that directory are automatically loaded.
+
+    config.middleware.insert 0, BotKiller
   end
 end

lib/bot_killer.rb

@@ -0,0 +1,20 @@
+class BotKiller
+  BLACKLISTED_BOTS = %r{
+    FAST    |
+    MauiBot
+  }xi
+
+  def initialize(app)
+    @app = app
+  end
+
+  def call(env)
+    request = Rack::Request.new(env)
+
+    if request.user_agent =~ BLACKLISTED_BOTS
+      [404, {}, ["Not found"]]
+    else
+      @app.call(env)
+    end
+  end
+end

test/integration/bot_killer_test.rb

@@ -0,0 +1,18 @@
+require 'test_helper'
+
+class BotKillerTest < ActionDispatch::IntegrationTest
+  test 'MauiBot is blacklisted' do
+    get '/', headers: { 'User-Agent' => 'MauiBot ([email protected])' }
+    assert_response :not_found
+  end
+
+  test 'FAST Enterprise is blacklisted' do
+    get '/', headers: { 'User-Agent' => 'FAST Enterprise Crawler/5.3.4 ([email protected])' }
+    assert_response :not_found
+  end
+
+  test 'other user agests are not blacklisted' do
+    get '/', headers: { 'User-Agent' => 'Foo' }
+    assert_response :ok
+  end
+end