Encode non-ASCII Column Names to UTF-8 (#531)

* need to convert the column names to UTF-8
* throw correctly encoded error message when it contains nanodbc error or sql
* add docs on the encoding of database and driver
* offload work to output_encoder

---------

Co-authored-by: shrektan <[email protected]>
Co-authored-by: Hadley Wickham <[email protected]>
Co-authored-by: Simon P. Couch <[email protected]>

Author: 4 people

NEWS.md

@@ -1,5 +1,11 @@
 # odbc (development version)
 
+* odbc now always converts the encoding of non-ASCII column names of the SQL
+  results to UTF-8. (@shrektan, #430)
+
+* Fixed issue that odbc may throw errors with garbage letters when the encoding
+  of client and db-server are different. (@shrektan, #432)
+
 * dbListFields: Now works with DBI::Id and DBI::SQL identifiers.
 
 * Transitioned `odbcDataType()` to use S4 for consistency. S3 methods defined

src/Iconv.cpp

@@ -8,7 +8,7 @@ using namespace Rcpp;
 #include "Iconv.h"
 
 Iconv::Iconv(const std::string& from, const std::string& to) {
-  if (from == "UTF-8") {
+  if (from.empty() || from == to) {
     cd_ = NULL;
   } else {
     cd_ = Riconv_open(to.c_str(), from.c_str());

src/odbc_result.cpp

@@ -53,7 +53,7 @@ void odbc_result::execute() {
       num_columns_ = r_->columns();
     } catch (const nanodbc::database_error& e) {
       c_->set_current_result(nullptr);
-      throw odbc_error(e, sql_);
+      throw odbc_error(e, sql_, output_encoder_);
     } catch (...) {
       c_->set_current_result(nullptr);
       throw;
@@ -462,7 +462,13 @@ std::vector<std::string> odbc_result::column_names(nanodbc::result const& r) {
   std::vector<std::string> names;
   names.reserve(num_columns_);
   for (short i = 0; i < num_columns_; ++i) {
-    names.push_back(r.column_name(i));
+    nanodbc::string_type name = r.column_name(i);
+    // We expect column names to share the same encoding as the
+    // data itself.  Similar to the handling of string fields,
+    // convert to UTF-8 before returning to user ( if needed )
+    names.push_back(
+        output_encoder_.makeString(name.c_str(), name.c_str() + name.length())
+    );
   }
   return names;
 }
@@ -831,11 +837,7 @@ void odbc_result::assign_string(
     if (value.is_null(column)) {
       res = NA_STRING;
     } else {
-      if (c_->encoding() != "") {
-        res = output_encoder_.makeSEXP(str.c_str(), str.c_str() + str.length());
-      } else { // If no encoding specified assume it is UTF-8 / ASCII
-        res = Rf_mkCharCE(str.c_str(), CE_UTF8);
-      }
+      res = output_encoder_.makeSEXP(str.c_str(), str.c_str() + str.length());
     }
   }
   SET_STRING_ELT(out[column], row, res);

src/odbc_result.h

@@ -19,13 +19,21 @@ inline void signal_unknown_field_type(short type, const std::string& name) {
 
 class odbc_error : public Rcpp::exception {
 public:
-  odbc_error(const nanodbc::database_error& e, const std::string& sql)
+  odbc_error(
+      const nanodbc::database_error& e,
+      const std::string& sql,
+      Iconv& output_encoder)
       : Rcpp::exception("", false) {
-    message = std::string(e.what()) + "\n<SQL> '" + sql + "'";
+    std::string m = std::string(e.what()) + "\n<SQL> '" + sql + "'";
+    // #432: [R] expects UTF-8 encoded strings but both nanodbc and sql are
+    // encoded in the database encoding, which may differ from UTF-8
+    message = Rf_translateChar(
+        output_encoder.makeSEXP(m.c_str(), m.c_str() + m.length()));
   }
   const char* what() const NANODBC_NOEXCEPT { return message.c_str(); }
 
 private:
+  // #432: must be native encoded, as R expects native encoded chars for error msg
   std::string message;
 };