initial node manager / watcher implementation

Author: ryanlecompte

.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Ryan LeCompte
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -0,0 +1,29 @@
+# RedisFailover
+
+Redis Failover provides a full automatic master/slave failover solution for Ruby
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'redis_failover'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install redis_failover
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

Rakefile

@@ -0,0 +1,9 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = %w(--format progress)
+end
+
+task :default => [:spec]

bin/redis_failover_server

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require 'redis_failover'
+
+manager = RedisFailover::NodeManager.new(:host => 'localhost', :password => 'fu')
+manager.start
+

lib/redis_failover.rb

@@ -0,0 +1,15 @@
+require 'redis'
+require 'thread'
+require 'logger'
+require 'securerandom'
+
+require 'redis_failover/cli'
+require 'redis_failover/util'
+require 'redis_failover/node'
+require 'redis_failover/errors'
+require 'redis_failover/client'
+require 'redis_failover/server'
+require 'redis_failover/version'
+require 'redis_failover/node_manager'
+require 'redis_failover/node_watcher'
+require 'redis_failover/configuration'

lib/redis_failover/cli.rb

@@ -0,0 +1,4 @@
+module RedisFailover
+  class CLI
+  end
+end

lib/redis_failover/client.rb

@@ -0,0 +1,4 @@
+module RedisFailover
+  class Client
+  end
+end

lib/redis_failover/configuration.rb

@@ -0,0 +1,4 @@
+module RedisFailover
+  class Configuration
+  end
+end

lib/redis_failover/errors.rb

@@ -0,0 +1,9 @@
+module RedisFailover
+  class Error < StandardError; end
+
+  class InvalidNodeError < Error; end
+
+  class NodeUnreachableError < Error; end
+
+  class NoMasterError < Error; end
+end

lib/redis_failover/node.rb

@@ -0,0 +1,90 @@
+module RedisFailover
+  # Represents a redis node (master or slave).
+  class Node
+    include Util
+
+    attr_reader :host, :port
+
+    def initialize(manager, options = {})
+      @host = options.fetch(:host) { raise InvalidNodeError, 'missing host'}
+      @port = options.fetch(:port, 6379)
+      @password = options[:password]
+      @manager = manager
+    end
+
+    def reachable?
+      fetch_info
+      true
+    rescue
+      false
+    end
+
+    def unreachable?
+      !reachable?
+    end
+
+    def master?
+      role == 'master'
+    end
+
+    def slave?
+      !master?
+    end
+
+    def wait_until_unreachable
+      redis.blpop(wait_key, 0)
+    rescue
+      unless reachable?
+        raise NodeUnreachableError, 'failed while waiting'
+      end
+    end
+
+    def stop_waiting
+      redis.lpush(wait_key, '1')
+    end
+
+    def make_slave!
+      master = @manager.current_master
+      redis.slaveof(master.host, master.port)
+    end
+
+    def make_master!
+      redis.slaveof('no', 'one')
+    end
+
+    def inspect
+      "<RedisFailover::Node #{to_s}>"
+    end
+
+    def to_s
+      "#{@host}:#{@port}"
+    end
+
+    def ==(other)
+      return false unless other.is_a?(Node)
+      return true if self.equal?(other)
+      [host, port] == [other.host, other.port]
+    end
+    alias_method :eql?, :==
+
+    private
+
+    def role
+      fetch_info[:role]
+    end
+
+    def fetch_info
+      symbolize_keys(redis.info)
+    end
+
+    def wait_key
+      @wait_key ||= SecureRandom.hex(32)
+    end
+
+    def redis
+      Redis.new(:host => @host, :password => @password, :port => @port)
+    rescue
+      raise NodeUnreachableError, 'failed to create redis client'
+    end
+  end
+end

lib/redis_failover/node_manager.rb

@@ -0,0 +1,119 @@
+module RedisFailover
+  # NodeManager manages a list of redis nodes.
+  class NodeManager
+    include Util
+
+    def initialize(*nodes)
+      @master, @slaves = parse_nodes(nodes)
+      @unreachable = []
+      @queue = Queue.new
+      @lock = Mutex.new
+    end
+
+    def start
+      trap_signals
+      spawn_watchers
+
+      logger.info('Redis Failover Server started successfully.')
+      while node = @queue.pop
+        if node.unreachable?
+          handle_unreachable(node)
+        elsif node.reachable?
+          handle_reachable(node)
+        end
+      end
+    end
+
+    def notify_state_change(node)
+      @queue << node
+    end
+
+    def current_master
+      @master
+    end
+
+    def nodes
+      @lock.synchronize do
+        {
+          :master => current_master.to_s,
+          :slaves => @slaves.map(&:to_s)
+        }
+      end
+    end
+
+    def shutdown
+      logger.info('Shutting down ...')
+      @watchers.each(&:shutdown)
+      exit(0)
+    end
+
+    private
+
+    def handle_unreachable(node)
+      @lock.synchronize do
+        # no-op if we already know about this node
+        return if @unreachable.include?(node)
+        logger.info("Handling unreachable node: #{node}")
+
+        # find a new master if this node was a master
+        if node == @master
+          logger.info("Demoting currently unreachable master #{node}.")
+          promote_new_master
+        end
+        @unreachable << node
+      end
+    end
+
+    def handle_reachable(node)
+      @lock.synchronize do
+        # no-op if we already know about this node
+        return if @master == node || @slaves.include?(node)
+        logger.info("Handling reachable node: #{node}")
+
+        @unreachable.delete(node)
+        @slaves << node
+        if current_master
+          # master already exists, make a slave
+          node.make_slave!
+        else
+          # no master exists, make this the new master
+          promote_new_master
+        end
+      end
+    end
+
+    def promote_new_master
+      @master = nil
+
+      if @slaves.empty?
+        logger.error('Failed to promote a new master since no slaves available.')
+        return
+      else
+        # make a slave the new master
+        node = @slaves.pop
+        node.make_master!
+        @master = node
+        logger.info("Successfully promoted #{@master} to master.")
+      end
+    end
+
+    def parse_nodes(nodes)
+      nodes = nodes.map { |opts| Node.new(self, opts) }
+      raise NoMasterError unless master = nodes.find(&:master?)
+      [master, nodes - [master]]
+    end
+
+    def spawn_watchers
+      @watchers = [@master, *@slaves].map do |node|
+          NodeWatcher.new(self, node)
+      end
+      @watchers.each(&:watch)
+    end
+
+    def trap_signals
+      %w(INT TERM).each do |signal|
+        trap(signal) { shutdown }
+      end
+    end
+  end
+end

lib/redis_failover/node_watcher.rb

@@ -0,0 +1,36 @@
+module RedisFailover
+  # Watches a specific redis node for its reachability.
+  class NodeWatcher
+    def initialize(manager, node)
+      @manager = manager
+      @node = node
+      @monitor_thread = nil
+      @done = false
+    end
+
+    def watch
+      @monitor_thread = Thread.new { monitor_node }
+    end
+
+    def shutdown
+      @done = true
+      @node.stop_waiting
+      @monitor_thread.join if @monitor_thread
+    end
+
+    private
+
+    def monitor_node
+      return if @done
+      @manager.notify_state_change(@node) if @node.reachable?
+      @node.wait_until_unreachable
+    rescue
+      @manager.notify_state_change(@node)
+      relax && retry
+    end
+
+    def relax
+      sleep(5)
+    end
+  end
+end

lib/redis_failover/server.rb

@@ -0,0 +1,4 @@
+module RedisFailover
+  class Server
+  end
+end