Merge pull request #471 from Ghabry/no-iconv

Remove Iconv support

Author: fdelapena

CMakeLists.txt

@@ -8,7 +8,7 @@ project(liblcf VERSION 0.8 LANGUAGES CXX)
 
 # Compilation options
 option(BUILD_SHARED_LIBS "Build shared library, disable for building the static library (default: ON)" ON)
-option(LIBLCF_WITH_ICU "ICU encoding detection (when OFF fallback to iconv, not recommended, default: ON)" ON)
+option(LIBLCF_WITH_ICU "ICU encoding handling (when disabled only windows-1252 is supported, default: ON)" ON)
 option(LIBLCF_WITH_XML "XML reading support (expat, default: ON)" ON)
 option(LIBLCF_UPDATE_MIMEDB "Whether to run update-mime-database after install (default: ON)" ON)
 option(LIBLCF_ENABLE_TOOLS "Whether to build the tools (default: ON)" ON)
@@ -343,16 +343,13 @@ set_property(TARGET lcf PROPERTY WINDOWS_EXPORT_ALL_SYMBOLS ON)
 # Name of the exported library
 set_property(TARGET lcf PROPERTY EXPORT_NAME liblcf)
 
-# icu or fallback to iconv
+# icu
 set(LCF_SUPPORT_ICU 0)
 if(LIBLCF_WITH_ICU)
 	find_package(ICU COMPONENTS i18n uc data REQUIRED)
 	target_link_libraries(lcf ICU::i18n ICU::uc ICU::data)
 	list(APPEND LIBLCF_DEPS "icu-i18n")
 	set(LCF_SUPPORT_ICU 1)
-else()
-	find_package(Iconv REQUIRED)
-	target_link_libraries(lcf Iconv::Iconv)
 endif()
 
 # expat

builds/cmake/Modules/FindIconv.cmake

@@ -4,8 +4,6 @@ include(CMakeFindDependencyMacro)
 
 if(@LCF_SUPPORT_ICU@)
 	find_dependency(ICU COMPONENTS i18n uc data REQUIRED)
-else()
-	find_dependency(Iconv REQUIRED)
 endif()
 
 if(@LCF_SUPPORT_XML@)

builds/cmake/liblcf-config.cmake.in

@@ -22,7 +22,7 @@ AM_CONDITIONAL(CROSS_COMPILING,[test "x$cross_compiling" = "xyes"])
 
 # Checks for libraries.
 AC_SUBST([LCF_SUPPORT_ICU],[0])
-AC_ARG_ENABLE([icu],[AS_HELP_STRING([--disable-icu],[Disable ICU encoding detection (fallback to iconv) [default=no]])])
+AC_ARG_ENABLE([icu],[AS_HELP_STRING([--disable-icu],[Disable ICU encoding handling (only windows-1252 supported) [default=no]])])
 AS_IF([test "x$enable_icu" != "xno"],[
 	AX_PKG_CHECK_MODULES([ICU],[],[icu-i18n],[LCF_SUPPORT_ICU=1])
 ])

configure.ac

@@ -13,23 +13,17 @@
 #include "lcf/scope_guard.h"
 #include <cstdio>
 #include <cstdlib>
-#include <exception>
 
 #if LCF_SUPPORT_ICU
 #   include <unicode/ucsdet.h>
 #   include <unicode/ucnv.h>
 #else
-#   ifdef _MSC_VER
-#       error MSVC builds require ICU
-#   endif
+#   include <cstdint>
 #endif
 
 #ifdef _WIN32
 #   include <windows.h>
 #else
-#   if !LCF_SUPPORT_ICU
-#       include <iconv.h>
-#   endif
 #   include <locale>
 #endif
 
@@ -82,12 +76,12 @@ void Encoder::Init() {
 		return;
 	}
 
-#if LCF_SUPPORT_ICU
 	auto code_page = atoi(_encoding.c_str());
 	const auto& storage_encoding = code_page > 0
 		? ReaderUtil::CodepageToEncoding(code_page)
 		: _encoding;
 
+#if LCF_SUPPORT_ICU
 	auto status = U_ZERO_ERROR;
 	constexpr auto runtime_encoding = "UTF-8";
 	auto conv_runtime = ucnv_open(runtime_encoding, &status);
@@ -111,26 +105,30 @@ void Encoder::Init() {
 	_conv_runtime = conv_runtime;
 	_conv_storage = conv_storage;
 #else
-	_conv_runtime = const_cast<char*>("UTF-8");
-	_conv_storage = const_cast<char*>(_encoding.c_str());
+	if (storage_encoding != "windows-1252") {
+		return;
+	}
+
+	_conv_runtime = 65001;
+	_conv_storage = 1252;
 #endif
 }
 
-void Encoder::Reset() {
 #if LCF_SUPPORT_ICU
-	auto* conv = reinterpret_cast<UConverter*>(_conv_runtime);
-	if (conv) ucnv_close(conv);
-	conv = reinterpret_cast<UConverter*>(_conv_storage);
-	if (conv) ucnv_close(conv);
-#endif
-}
+void Encoder::Reset() {
+	if (_conv_runtime) {
+		ucnv_close(_conv_runtime);
+		_conv_runtime = nullptr;
+	}
 
+	if (_conv_storage) {
+		ucnv_close(_conv_storage);
+		_conv_storage = nullptr;
+	}
+}
 
-void Encoder::Convert(std::string& str, void* conv_dst_void, void* conv_src_void) {
-#if LCF_SUPPORT_ICU
+void Encoder::Convert(std::string& str, UConverter* conv_dst, UConverter* conv_src) {
 	const auto& src = str;
-	auto* conv_dst = reinterpret_cast<UConverter*>(conv_dst_void);
-	auto* conv_src = reinterpret_cast<UConverter*>(conv_src_void);
 
 	auto status = U_ZERO_ERROR;
 	_buffer.resize(src.size() * 4);
@@ -151,36 +149,65 @@ void Encoder::Convert(std::string& str, void* conv_dst_void, void* conv_src_void
 	}
 
 	str.assign(_buffer.data(), dst_p);
-	return;
+}
 #else
-	auto* conv_dst = reinterpret_cast<const char*>(conv_dst_void);
-	auto* conv_src = reinterpret_cast<const char*>(conv_src_void);
-	iconv_t cd = iconv_open(conv_dst, conv_src);
-	if (cd == (iconv_t)-1)
-		return;
-	char *src = &str.front();
-	size_t src_left = str.size();
-	size_t dst_size = str.size() * 5 + 10;
-	_buffer.resize(dst_size);
-	char *dst = _buffer.data();
-	size_t dst_left = dst_size;
-#    ifdef ICONV_CONST
-	char ICONV_CONST *p = src;
-#    else
-	char *p = src;
-#    endif
-	char *q = dst;
-	size_t status = iconv(cd, &p, &src_left, &q, &dst_left);
-	iconv_close(cd);
-	if (status == (size_t) -1 || src_left > 0) {
-		str.clear();
+void Encoder::Convert(std::string& str, int conv_dst, int) {
+	if (str.empty()) {
 		return;
 	}
-	*q++ = '\0';
-	str.assign(dst, dst_size - dst_left);
-	return;
-#endif
+
+	size_t buf_idx = 0;
+
+	if (conv_dst == 65001) {
+		// From 1252 to UTF-8
+		// Based on https://stackoverflow.com/q/4059775/
+		_buffer.resize(str.size() * 2 + 1);
+
+		for (unsigned char ch: str) {
+			if (ch < 0x80) {
+				_buffer[buf_idx] = static_cast<char>(ch);
+			} else {
+				_buffer[buf_idx] = static_cast<char>(0xC0 | (ch >> 6));
+				++buf_idx;
+				_buffer[buf_idx] = static_cast<char>(0x80 | (ch & 0x3F));
+			}
+
+			++buf_idx;
+		}
+	} else {
+		// From UTF-8 to 1252
+		// Based on https://stackoverflow.com/q/23689733/
+		_buffer.resize(str.size() + 1);
+		uint32_t codepoint;
+
+		for (size_t str_idx = 0; str_idx < str.size(); ++str_idx) {
+			unsigned char ch = str[str_idx];
+			if (ch <= 0x7F) {
+				codepoint = ch;
+			} else if (ch <= 0xBF) {
+				codepoint = (codepoint << 6) | (ch & 0x3F);
+			} else if (ch <= 0xDF) {
+				codepoint = ch & 0x1F;
+			} else if (ch <= 0xEF) {
+				codepoint = ch & 0x0F;
+			} else {
+				codepoint = ch & 0x07;
+			}
+			++str_idx;
+			ch = str[str_idx];
+			if (((ch & 0xC0) != 0x80) && (codepoint <= 0x10ffff)) {
+				if (codepoint <= 255) {
+					_buffer[buf_idx] = static_cast<char>(codepoint);
+				} else {
+					_buffer[buf_idx] = '?';
+				}
+			}
+			++buf_idx;
+		}
+	}
+
+	str.assign(_buffer.data(), buf_idx);
 }
+#endif
 
 } //namespace lcf
-