Merge pull request #88 from brain-geek/master

Change algorithm for detecting separator

Author: pcreux

lib/csv_importer/csv_reader.rb

@@ -53,7 +53,18 @@ def sanitize_content(csv_content)
     SEPARATORS = [",", ";", "\t"]
 
     def detect_separator(csv_content)
-      SEPARATORS.sort_by { |separator| csv_content.count(separator) }.last
+      SEPARATORS.min_by do |separator|
+        csv_content.count(separator)
+
+        all_lines = csv_content.lines
+        base_number = all_lines.first.count(separator)
+
+        if base_number.zero?
+          Float::MAX
+        else
+          all_lines.map{|line| line.count(separator) - base_number }.map(&:abs).inject(0) { |sum, i| sum + i }
+        end
+      end
     end
 
     # Remove trailing white spaces and ensure we always return a string

spec/csv_importer/csv_reader_spec.rb

@@ -24,6 +24,12 @@ module CSVImporter
       expect(reader.header).to eq ["email", "first_name", "last_name"]
     end
 
+    it "supports semicolon separated csv when content has lot of commas" do
+      reader = CSVReader.new(content: "email;first_name;last_name;letter_ids\n
+                                      [email protected];Peter;Stone;1,2,3,4,5,6,7,8,9,10,11,12,13,14")
+      expect(reader.header).to eq ["email", "first_name", "last_name", "letter_ids"]
+    end
+
     it "supports tab separated csv" do
       reader = CSVReader.new(content: "email\tfirst_name\tlast_name")
       expect(reader.header).to eq ["email", "first_name", "last_name"]