From c0a22046c73d55ee77bbb476270add1addeb348c Mon Sep 17 00:00:00 2001 From: Santiago Perez Perret Date: Sun, 30 Aug 2020 11:56:48 -0300 Subject: [PATCH 1/6] Update default reader encoding to be compatible with ruby 2.7.1 --- lib/csv_importer/csv_reader.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/csv_importer/csv_reader.rb b/lib/csv_importer/csv_reader.rb index 22f690d..9c2596d 100644 --- a/lib/csv_importer/csv_reader.rb +++ b/lib/csv_importer/csv_reader.rb @@ -8,7 +8,7 @@ class CSVReader attribute :file # IO attribute :path, String attribute :quote_char, String, default: '"' - attribute :encoding, String, default: 'UTF-8:UTF-8' + attribute :encoding, String, default: 'UTF-8' def csv_rows @csv_rows ||= begin From c2b22a6451239e82464b51309bd604c48af105d8 Mon Sep 17 00:00:00 2001 From: Santiago Perez Perret Date: Sun, 30 Aug 2020 12:00:32 -0300 Subject: [PATCH 2/6] Change CSVReader#sanitize content to use keyword arguments directly (ruby 2.7 friendly) --- lib/csv_importer/csv_reader.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/csv_importer/csv_reader.rb b/lib/csv_importer/csv_reader.rb index 9c2596d..09ef516 100644 --- a/lib/csv_importer/csv_reader.rb +++ b/lib/csv_importer/csv_reader.rb @@ -46,7 +46,7 @@ def read_content def sanitize_content(csv_content) internal_encoding = encoding.split(':').last csv_content - .encode(Encoding.find(internal_encoding), {invalid: :replace, undef: :replace, replace: ''}) # Remove invalid byte sequences + .encode(Encoding.find(internal_encoding), invalid: :replace, undef: :replace, replace: '') # Remove invalid byte sequences .gsub(/\r\r?\n?/, "\n") # Replaces windows line separators with "\n" end From 7939d50f857c240754c9a47d7948edfb0475e41a Mon Sep 17 00:00:00 2001 From: Santiago Perez Perret Date: Mon, 31 Aug 2020 10:58:01 -0300 Subject: [PATCH 3/6] Re-structure dependencies for dev env to work --- .gitignore | 1 - Gemfile | 6 --- Gemfile.lock | 111 +++++++++++++++++++++++++++++++++++++++++++ csv-importer.gemspec | 8 +++- 4 files changed, 117 insertions(+), 9 deletions(-) create mode 100644 Gemfile.lock diff --git a/.gitignore b/.gitignore index 35af4e4..0e10a10 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ /.bundle/ /.yardoc -/Gemfile.lock /_yardoc/ /coverage/ /doc/ diff --git a/Gemfile b/Gemfile index 50e80a4..16467ff 100644 --- a/Gemfile +++ b/Gemfile @@ -2,9 +2,3 @@ source 'https://rubygems.org' # Specify your gem's dependencies in csv-importer.gemspec gemspec - -gem 'rspec', '~> 3.3.0' -gem 'guard-rspec' -gem 'activemodel' -gem 'simplecov', require: nil -gem "codeclimate-test-reporter", require: nil diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..ad5b76b --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,111 @@ +PATH + remote: . + specs: + csv-importer (0.7.0) + virtus + +GEM + remote: https://rubygems.org/ + specs: + activemodel (5.2.4.3) + activesupport (= 5.2.4.3) + activesupport (5.2.4.3) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 0.7, < 2) + minitest (~> 5.1) + tzinfo (~> 1.1) + axiom-types (0.1.1) + descendants_tracker (~> 0.0.4) + ice_nine (~> 0.11.0) + thread_safe (~> 0.3, >= 0.3.1) + codeclimate-test-reporter (1.0.9) + simplecov (<= 0.13) + coderay (1.1.3) + coercible (1.0.0) + descendants_tracker (~> 0.0.1) + concurrent-ruby (1.1.7) + descendants_tracker (0.0.4) + thread_safe (~> 0.3, >= 0.3.1) + diff-lcs (1.4.4) + docile (1.1.5) + equalizer (0.0.11) + ffi (1.13.1) + formatador (0.2.5) + guard (2.16.2) + formatador (>= 0.2.4) + listen (>= 2.7, < 4.0) + lumberjack (>= 1.0.12, < 2.0) + nenv (~> 0.1) + notiffany (~> 0.0) + pry (>= 0.9.12) + shellany (~> 0.0) + thor (>= 0.18.1) + guard-compat (1.2.1) + guard-rspec (4.7.3) + guard (~> 2.1) + guard-compat (~> 1.1) + rspec (>= 2.99.0, < 4.0) + i18n (1.8.5) + concurrent-ruby (~> 1.0) + ice_nine (0.11.2) + json (2.3.1) + listen (3.2.1) + rb-fsevent (~> 0.10, >= 0.10.3) + rb-inotify (~> 0.9, >= 0.9.10) + lumberjack (1.2.7) + method_source (1.0.0) + minitest (5.14.1) + nenv (0.3.0) + notiffany (0.1.3) + nenv (~> 0.1) + shellany (~> 0.0) + pry (0.13.1) + coderay (~> 1.1) + method_source (~> 1.0) + rake (13.0.1) + rb-fsevent (0.10.4) + rb-inotify (0.10.1) + ffi (~> 1.0) + rspec (3.3.0) + rspec-core (~> 3.3.0) + rspec-expectations (~> 3.3.0) + rspec-mocks (~> 3.3.0) + rspec-core (3.3.2) + rspec-support (~> 3.3.0) + rspec-expectations (3.3.1) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.3.0) + rspec-mocks (3.3.2) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.3.0) + rspec-support (3.3.0) + shellany (0.0.1) + simplecov (0.13.0) + docile (~> 1.1.0) + json (>= 1.8, < 3) + simplecov-html (~> 0.10.0) + simplecov-html (0.10.2) + thor (1.0.1) + thread_safe (0.3.6) + tzinfo (1.2.7) + thread_safe (~> 0.1) + virtus (1.0.5) + axiom-types (~> 0.1) + coercible (~> 1.0) + descendants_tracker (~> 0.0, >= 0.0.3) + equalizer (~> 0.0, >= 0.0.9) + +PLATFORMS + ruby + +DEPENDENCIES + activemodel (~> 5) + codeclimate-test-reporter + csv-importer! + guard-rspec + rake + rspec (~> 3.3.0) + simplecov + +BUNDLED WITH + 2.0.2 diff --git a/csv-importer.gemspec b/csv-importer.gemspec index fd9139a..7644368 100644 --- a/csv-importer.gemspec +++ b/csv-importer.gemspec @@ -20,6 +20,10 @@ Gem::Specification.new do |spec| spec.add_dependency "virtus" - spec.add_development_dependency "bundler", "~> 1.8" - spec.add_development_dependency "rake", "~> 10.0" + spec.add_development_dependency "rspec", "~> 3.3.0" + spec.add_development_dependency "rake" + spec.add_development_dependency "guard-rspec" + spec.add_development_dependency "activemodel", "~> 5" + spec.add_development_dependency "simplecov" + spec.add_development_dependency "codeclimate-test-reporter" end From 5de0b02c832551325c7358c371cd0af7f8ec3aa6 Mon Sep 17 00:00:00 2001 From: Santiago Perez Perret Date: Mon, 31 Aug 2020 11:09:50 -0300 Subject: [PATCH 4/6] Add ruby v2.7 to CI --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index a451fb1..98eb6bf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ language: ruby rvm: - 2.3.0 + - 2.7 addons: code_climate: repo_token: bcecbf1b229a2ddd666a2c3830f26a0113fd56ae1586d30d2d3fb1af837bf0e4 From 39e9c63836f312c474eb66a21a9ed8e86702a37e Mon Sep 17 00:00:00 2001 From: Santiago Perez Perret Date: Mon, 31 Aug 2020 12:36:39 -0300 Subject: [PATCH 5/6] Make encoding clearer --- Gemfile.lock | 2 +- lib/csv_importer/csv_reader.rb | 29 ++++++++++++++++++++++++----- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index ad5b76b..5db9a9b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -108,4 +108,4 @@ DEPENDENCIES simplecov BUNDLED WITH - 2.0.2 + 2.1.4 diff --git a/lib/csv_importer/csv_reader.rb b/lib/csv_importer/csv_reader.rb index 09ef516..c5ece38 100644 --- a/lib/csv_importer/csv_reader.rb +++ b/lib/csv_importer/csv_reader.rb @@ -8,14 +8,18 @@ class CSVReader attribute :file # IO attribute :path, String attribute :quote_char, String, default: '"' - attribute :encoding, String, default: 'UTF-8' + attribute :encoding, String, default: 'UTF-8:UTF-8' def csv_rows @csv_rows ||= begin sane_content = sanitize_content(read_content) separator = detect_separator(sane_content) - cells = CSV.parse(sane_content, col_sep: separator, quote_char: quote_char, skip_blanks: true, encoding: encoding) - sanitize_cells(cells) + cells = CSV.parse( + sane_content, + col_sep: separator, quote_char: quote_char, skip_blanks: true, + external_encoding: source_encoding + ) + sanitize_cells(encode_cells(cells)) end end @@ -44,9 +48,8 @@ def read_content end def sanitize_content(csv_content) - internal_encoding = encoding.split(':').last csv_content - .encode(Encoding.find(internal_encoding), invalid: :replace, undef: :replace, replace: '') # Remove invalid byte sequences + .encode(Encoding.find(source_encoding), invalid: :replace, undef: :replace, replace: '') # Remove invalid byte sequences .gsub(/\r\r?\n?/, "\n") # Replaces windows line separators with "\n" end @@ -75,5 +78,21 @@ def sanitize_cells(rows) end end end + + def encode_cells(rows) + rows.map do |cells| + cells.map do |cell| + cell ? cell.encode(target_encoding) : "" + end + end + end + + def source_encoding + encoding.split(':').first || 'UTF-8' + end + + def target_encoding + encoding.split(':').last || 'UTF-8' + end end end From 95dbf58dcc72dc951be2c0921e01060733c21aae Mon Sep 17 00:00:00 2001 From: Santiago Perez Perret Date: Mon, 31 Aug 2020 12:45:30 -0300 Subject: [PATCH 6/6] Remove Gemfile.lock again --- .gitignore | 1 + Gemfile.lock | 111 --------------------------------------------------- 2 files changed, 1 insertion(+), 111 deletions(-) delete mode 100644 Gemfile.lock diff --git a/.gitignore b/.gitignore index 0e10a10..35af4e4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /.bundle/ /.yardoc +/Gemfile.lock /_yardoc/ /coverage/ /doc/ diff --git a/Gemfile.lock b/Gemfile.lock deleted file mode 100644 index 5db9a9b..0000000 --- a/Gemfile.lock +++ /dev/null @@ -1,111 +0,0 @@ -PATH - remote: . - specs: - csv-importer (0.7.0) - virtus - -GEM - remote: https://rubygems.org/ - specs: - activemodel (5.2.4.3) - activesupport (= 5.2.4.3) - activesupport (5.2.4.3) - concurrent-ruby (~> 1.0, >= 1.0.2) - i18n (>= 0.7, < 2) - minitest (~> 5.1) - tzinfo (~> 1.1) - axiom-types (0.1.1) - descendants_tracker (~> 0.0.4) - ice_nine (~> 0.11.0) - thread_safe (~> 0.3, >= 0.3.1) - codeclimate-test-reporter (1.0.9) - simplecov (<= 0.13) - coderay (1.1.3) - coercible (1.0.0) - descendants_tracker (~> 0.0.1) - concurrent-ruby (1.1.7) - descendants_tracker (0.0.4) - thread_safe (~> 0.3, >= 0.3.1) - diff-lcs (1.4.4) - docile (1.1.5) - equalizer (0.0.11) - ffi (1.13.1) - formatador (0.2.5) - guard (2.16.2) - formatador (>= 0.2.4) - listen (>= 2.7, < 4.0) - lumberjack (>= 1.0.12, < 2.0) - nenv (~> 0.1) - notiffany (~> 0.0) - pry (>= 0.9.12) - shellany (~> 0.0) - thor (>= 0.18.1) - guard-compat (1.2.1) - guard-rspec (4.7.3) - guard (~> 2.1) - guard-compat (~> 1.1) - rspec (>= 2.99.0, < 4.0) - i18n (1.8.5) - concurrent-ruby (~> 1.0) - ice_nine (0.11.2) - json (2.3.1) - listen (3.2.1) - rb-fsevent (~> 0.10, >= 0.10.3) - rb-inotify (~> 0.9, >= 0.9.10) - lumberjack (1.2.7) - method_source (1.0.0) - minitest (5.14.1) - nenv (0.3.0) - notiffany (0.1.3) - nenv (~> 0.1) - shellany (~> 0.0) - pry (0.13.1) - coderay (~> 1.1) - method_source (~> 1.0) - rake (13.0.1) - rb-fsevent (0.10.4) - rb-inotify (0.10.1) - ffi (~> 1.0) - rspec (3.3.0) - rspec-core (~> 3.3.0) - rspec-expectations (~> 3.3.0) - rspec-mocks (~> 3.3.0) - rspec-core (3.3.2) - rspec-support (~> 3.3.0) - rspec-expectations (3.3.1) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.3.0) - rspec-mocks (3.3.2) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.3.0) - rspec-support (3.3.0) - shellany (0.0.1) - simplecov (0.13.0) - docile (~> 1.1.0) - json (>= 1.8, < 3) - simplecov-html (~> 0.10.0) - simplecov-html (0.10.2) - thor (1.0.1) - thread_safe (0.3.6) - tzinfo (1.2.7) - thread_safe (~> 0.1) - virtus (1.0.5) - axiom-types (~> 0.1) - coercible (~> 1.0) - descendants_tracker (~> 0.0, >= 0.0.3) - equalizer (~> 0.0, >= 0.0.9) - -PLATFORMS - ruby - -DEPENDENCIES - activemodel (~> 5) - codeclimate-test-reporter - csv-importer! - guard-rspec - rake - rspec (~> 3.3.0) - simplecov - -BUNDLED WITH - 2.1.4