add test cases for harvester

- move DIRECTORY_LAYOUT global to OAI::Harvester::Harvest class and make configurable
- delegate record parsing to ListRecords response in harvester
- do nothing with harvested tempfile if no directory storage is configured
- use application exceptions in harvester

Author: barmintor

Diff for: Rakefile

@@ -34,6 +34,13 @@ namespace :test do
     t.warning = false
   end
 
+  Rake::TestTask.new('harvester') do |t|
+    t.libs << ['lib', 'test/harvester']
+    t.pattern = 'test/harvester/tc_*.rb'
+    #t.verbose = true
+    t.warning = false
+  end
+
   Rake::TestTask.new('provider') do |t|
     t.libs << ['lib', 'test/provider']
     t.pattern = 'test/provider/tc_*.rb'

Diff for: bin/oai

@@ -5,8 +5,6 @@
 
 require 'optparse'
 
-DIRECTORY_LAYOUT = "%Y/%m".freeze
-
 require 'oai/harvester'
 
 include OAI::Harvester

Diff for: lib/oai/harvester/harvest.rb

@@ -3,8 +3,8 @@
 
 module OAI
   module Harvester
-  
     class Harvest
+      DIRECTORY_LAYOUT = "%Y/%m".freeze
 
       def initialize(config = nil, directory = nil, date = nil)
         @config = config || Config.load
@@ -43,22 +43,27 @@ def harvest(site)
         # Allow a from date to be passed in
         opts[:from] = earliest(opts[:url]) unless opts[:from]
         opts.delete(:set) if 'all' == opts[:set]
-        
         begin
           # Connect, and download
           file, records = call(opts.delete(:url), opts)
 
-          # Move document to storage directory
-          dir = File.join(@directory, date_based_directory(harvest_time))
-          FileUtils.mkdir_p dir
-          FileUtils.mv(file.path, 
-            File.join(dir, "#{site}-#{filename(Time.parse(opts[:from]), 
-            harvest_time)}.xml.gz"))
+          # Move document to storage directory if configured
+          if @directory
+            directory_layout = @config.layouts[site] if @config.layouts
+            dir = File.join(@directory, date_based_directory(harvest_time, directory_layout))
+            FileUtils.mkdir_p dir
+            FileUtils.mv(file.path,
+              File.join(dir, "#{site}-#{filename(Time.parse(opts[:from]),
+              harvest_time)}.xml.gz"))
+          else
+            puts "no configured destination for temp file" if @interactive
+          end
           @config.sites[site]['last'] = harvest_time
-        rescue
-          raise $! unless $!.respond_to?(:code)
-          raise $! if not @interactive || "noRecordsMatch" != $!.code
-          puts "No new records available"
+        rescue OAI::NoMatchException
+          puts "No new records available" if @interactive
+        rescue OAI::Exception => ex
+          raise ex if not @interactive
+          puts ex.message
         end
       end
 
@@ -69,15 +74,15 @@ def call(url, opts)
         records = 0;
         client = OAI::Client.new(url, :parser => @parser)
         provider_config = client.identify
-        
+
         file = Tempfile.new('oai_data')
         gz = Zlib::GzipWriter.new(file)
         gz << "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n"
         gz << "<records>"
         begin
           response = client.list_records(options)
-          get_records(response.doc).each do |rec|
-            gz << rec
+          response.each do |rec|
+            gz << rec._source
             records += 1
           end
           puts "#{records} records retrieved" if @interactive
@@ -89,8 +94,8 @@ def call(url, opts)
             puts "\nresumption token recieved, continuing" if @interactive
             response = client.list_records(:resumption_token => 
               response.resumption_token)
-              get_records(response.doc).each do |rec|
-                gz << rec
+              response.each do |rec|
+                gz << rec._source
                 records += 1
               end
             puts "#{records} records retrieved" if @interactive
@@ -118,8 +123,9 @@ def build_options_hash(site)
         options
       end
 
-      def date_based_directory(time)
-        "#{time.strftime(DIRECTORY_LAYOUT)}"
+      def date_based_directory(time, directory_layout = nil)
+        directory_layout ||= Harvest::DIRECTORY_LAYOUT
+        "#{time.strftime(directory_layout)}"
       end
 
       def filename(from_time, until_time)

Diff for: test/harvester/tc_harvest.rb

@@ -0,0 +1,33 @@
+require 'test_helper_harvester'
+
+class HarvestTest < Test::Unit::TestCase
+  ONE_HOUR = 3600
+  EARLIEST_FIXTURE = "1998-05-02T04:00:00Z"
+  LATEST_FIXTURE = "2005-12-25T05:00:00Z"
+  def test_harvest
+    until_value = Time.now.utc - ONE_HOUR
+    config = OpenStruct.new(sites: { 'test' => { 'url' => 'http://localhost:3333/oai' }})
+    OAI::Harvester::Harvest.new(config).start
+    last = config.sites.dig('test', 'last')
+    assert_kind_of Time, last
+    assert last >= (until_value + ONE_HOUR), "#{last} < #{(until_value + ONE_HOUR)}"
+  end
+
+  def test_harvest_from_last
+    from_value = Time.parse(LATEST_FIXTURE).utc
+    now = Time.now.utc
+    config = OpenStruct.new(sites: { 'test' => { 'url' => 'http://localhost:3333/oai' }})
+    OAI::Harvester::Harvest.new(config, nil, from_value).start
+    last = config.sites.dig('test', 'last')
+    assert last >= now, "#{last} < #{now}"
+  end
+
+  def test_harvest_after_last
+    from_value = Time.parse(LATEST_FIXTURE).utc + 1
+    config = OpenStruct.new(sites: { 'test' => { 'url' => 'http://localhost:3333/oai' }})
+    OAI::Harvester::Harvest.new(config, nil, from_value).start
+    last = config.sites.dig('test', 'last')
+    assert_kind_of NilClass, last
+  end
+end
+

Diff for: test/harvester/test_helper_harvester.rb

@@ -0,0 +1,6 @@
+require 'oai'
+require 'oai/harvester'
+require 'test/unit'
+
+require File.dirname(__FILE__) + '/../client/helpers/provider'
+require File.dirname(__FILE__) + '/../client/helpers/test_wrapper'