diff --git a/.travis.yml b/.travis.yml index a451fb1..98eb6bf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ language: ruby rvm: - 2.3.0 + - 2.7 addons: code_climate: repo_token: bcecbf1b229a2ddd666a2c3830f26a0113fd56ae1586d30d2d3fb1af837bf0e4 diff --git a/Gemfile b/Gemfile index 50e80a4..16467ff 100644 --- a/Gemfile +++ b/Gemfile @@ -2,9 +2,3 @@ source 'https://rubygems.org' # Specify your gem's dependencies in csv-importer.gemspec gemspec - -gem 'rspec', '~> 3.3.0' -gem 'guard-rspec' -gem 'activemodel' -gem 'simplecov', require: nil -gem "codeclimate-test-reporter", require: nil diff --git a/csv-importer.gemspec b/csv-importer.gemspec index fd9139a..7644368 100644 --- a/csv-importer.gemspec +++ b/csv-importer.gemspec @@ -20,6 +20,10 @@ Gem::Specification.new do |spec| spec.add_dependency "virtus" - spec.add_development_dependency "bundler", "~> 1.8" - spec.add_development_dependency "rake", "~> 10.0" + spec.add_development_dependency "rspec", "~> 3.3.0" + spec.add_development_dependency "rake" + spec.add_development_dependency "guard-rspec" + spec.add_development_dependency "activemodel", "~> 5" + spec.add_development_dependency "simplecov" + spec.add_development_dependency "codeclimate-test-reporter" end diff --git a/lib/csv_importer/csv_reader.rb b/lib/csv_importer/csv_reader.rb index 22f690d..c5ece38 100644 --- a/lib/csv_importer/csv_reader.rb +++ b/lib/csv_importer/csv_reader.rb @@ -14,8 +14,12 @@ def csv_rows @csv_rows ||= begin sane_content = sanitize_content(read_content) separator = detect_separator(sane_content) - cells = CSV.parse(sane_content, col_sep: separator, quote_char: quote_char, skip_blanks: true, encoding: encoding) - sanitize_cells(cells) + cells = CSV.parse( + sane_content, + col_sep: separator, quote_char: quote_char, skip_blanks: true, + external_encoding: source_encoding + ) + sanitize_cells(encode_cells(cells)) end end @@ -44,9 +48,8 @@ def read_content end def sanitize_content(csv_content) - internal_encoding = encoding.split(':').last csv_content - .encode(Encoding.find(internal_encoding), {invalid: :replace, undef: :replace, replace: ''}) # Remove invalid byte sequences + .encode(Encoding.find(source_encoding), invalid: :replace, undef: :replace, replace: '') # Remove invalid byte sequences .gsub(/\r\r?\n?/, "\n") # Replaces windows line separators with "\n" end @@ -75,5 +78,21 @@ def sanitize_cells(rows) end end end + + def encode_cells(rows) + rows.map do |cells| + cells.map do |cell| + cell ? cell.encode(target_encoding) : "" + end + end + end + + def source_encoding + encoding.split(':').first || 'UTF-8' + end + + def target_encoding + encoding.split(':').last || 'UTF-8' + end end end