From d41345f33b1533d7102682ed65e452b77422b986 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1n=20Kobezda?= Date: Thu, 29 Sep 2016 18:34:58 +0200 Subject: [PATCH 1/2] Check for iconv() TRANSLIT support Some iconv() implementations don't support transliteration. When iconv with //TRANSLIT is first attempted and fails, try again without it, and remember the result. Fixes #364 --- include/utils.h | 1 + src/stflpp.cpp | 5 +++-- src/utils.cpp | 44 ++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/include/utils.h b/include/utils.h index e5293751..0f20f89f 100644 --- a/include/utils.h +++ b/include/utils.h @@ -46,6 +46,7 @@ class utils { static bool try_fs_lock(const std::string& lock_file, pid_t & pid); static void remove_fs_lock(const std::string& lock_file); + static std::string translit(const std::string& tocode, const std::string& fromcode); static std::string convert_text(const std::string& text, const std::string& tocode, const std::string& fromcode); static std::string get_command_output(const std::string& cmd); diff --git a/src/stflpp.cpp b/src/stflpp.cpp index fb6dcaea..0f47c44e 100644 --- a/src/stflpp.cpp +++ b/src/stflpp.cpp @@ -4,6 +4,7 @@ #include #include +#include namespace newsbeuter { @@ -16,7 +17,7 @@ namespace newsbeuter { */ stfl::form::form(const std::string& text) : f(0) { - ipool = stfl_ipool_create((std::string(nl_langinfo(CODESET)) + "//TRANSLIT").c_str()); + ipool = stfl_ipool_create(utils::translit(std::string(nl_langinfo(CODESET)), "WCHAR_T").c_str()); if (!ipool) { throw exception(errno); } @@ -82,7 +83,7 @@ static std::mutex quote_mtx; std::string stfl::quote(const std::string& text) { std::lock_guard lock(quote_mtx); - stfl_ipool * ipool = stfl_ipool_create((std::string(nl_langinfo(CODESET)) + "//TRANSLIT").c_str()); + stfl_ipool * ipool = stfl_ipool_create(utils::translit(std::string(nl_langinfo(CODESET)), "WCHAR_T").c_str()); std::string retval = stfl_ipool_fromwc(ipool,stfl_quote(stfl_ipool_towc(ipool,text.c_str()))); stfl_ipool_destroy(ipool); return retval; diff --git a/src/utils.cpp b/src/utils.cpp index 36e1013e..92bc6413 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -256,6 +256,46 @@ bool utils::try_fs_lock(const std::string& lock_file, pid_t & pid) { return false; } +std::string utils::translit(const std::string& tocode, const std::string& fromcode) +{ + std::string tlit = "//TRANSLIT"; + + typedef enum translit_state { + UNKNOWN, + SUPPORTED, + UNSUPPORTED + } translit_state_t; + + static translit_state_t state = UNKNOWN; + + // TRANSLIT is not needed when converting to unicode encodings + if (tocode == "utf-8" || tocode == "WCHAR_T") return tocode; + + if (state == UNKNOWN) { + iconv_t cd = ::iconv_open((tocode + "//TRANSLIT").c_str(), fromcode.c_str()); + + if (cd == reinterpret_cast(-1)) { + if (errno == EINVAL) { + iconv_t cd = ::iconv_open(tocode.c_str(), fromcode.c_str()); + if (cd != reinterpret_cast(-1)) { + state = UNSUPPORTED; + } else { + fprintf(stderr, "iconv_open('%s', '%s') failed: %s", tocode.c_str(), fromcode.c_str(), strerror(errno)); + abort(); + } + } else { + fprintf(stderr, "iconv_open('%s//TRANSLIT', '%s') failed: %s", tocode.c_str(), fromcode.c_str(), strerror(errno)); + abort(); + } + } else { + state = SUPPORTED; + } + + iconv_close(cd); + } + + return ((state == SUPPORTED) ? (tocode + tlit) : (tocode)); +} std::string utils::convert_text(const std::string& text, const std::string& tocode, const std::string& fromcode) { std::string result; @@ -263,7 +303,7 @@ std::string utils::convert_text(const std::string& text, const std::string& toco if (strcasecmp(tocode.c_str(), fromcode.c_str())==0) return text; - iconv_t cd = ::iconv_open((tocode + "//TRANSLIT").c_str(), fromcode.c_str()); + iconv_t cd = ::iconv_open(translit(tocode, fromcode).c_str(), fromcode.c_str()); if (cd == reinterpret_cast(-1)) return result; @@ -499,7 +539,7 @@ std::wstring utils::str2wstr(const std::string& str) { std::string utils::wstr2str(const std::wstring& wstr) { std::string codeset = nl_langinfo(CODESET); - codeset.append("//TRANSLIT"); + codeset = translit(codeset, "WCHAR_T"); struct stfl_ipool * ipool = stfl_ipool_create(codeset.c_str()); std::string result = stfl_ipool_fromwc(ipool, wstr.c_str()); stfl_ipool_destroy(ipool); From d0cea1b4c1c0968eb4b78e5c4ce2b57e37fc9a83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1n=20Kobezda?= Date: Thu, 29 Sep 2016 19:07:43 +0200 Subject: [PATCH 2/2] Overload utils::translit to accept const char* tocode --- include/utils.h | 1 + src/stflpp.cpp | 4 ++-- src/utils.cpp | 5 +++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/include/utils.h b/include/utils.h index 0f20f89f..6419edc5 100644 --- a/include/utils.h +++ b/include/utils.h @@ -46,6 +46,7 @@ class utils { static bool try_fs_lock(const std::string& lock_file, pid_t & pid); static void remove_fs_lock(const std::string& lock_file); + static std::string translit(const char* tocode, const std::string& fromcode); static std::string translit(const std::string& tocode, const std::string& fromcode); static std::string convert_text(const std::string& text, const std::string& tocode, const std::string& fromcode); diff --git a/src/stflpp.cpp b/src/stflpp.cpp index 0f47c44e..3a9da1a1 100644 --- a/src/stflpp.cpp +++ b/src/stflpp.cpp @@ -17,7 +17,7 @@ namespace newsbeuter { */ stfl::form::form(const std::string& text) : f(0) { - ipool = stfl_ipool_create(utils::translit(std::string(nl_langinfo(CODESET)), "WCHAR_T").c_str()); + ipool = stfl_ipool_create(utils::translit(nl_langinfo(CODESET), "WCHAR_T").c_str()); if (!ipool) { throw exception(errno); } @@ -83,7 +83,7 @@ static std::mutex quote_mtx; std::string stfl::quote(const std::string& text) { std::lock_guard lock(quote_mtx); - stfl_ipool * ipool = stfl_ipool_create(utils::translit(std::string(nl_langinfo(CODESET)), "WCHAR_T").c_str()); + stfl_ipool * ipool = stfl_ipool_create(utils::translit(nl_langinfo(CODESET), "WCHAR_T").c_str()); std::string retval = stfl_ipool_fromwc(ipool,stfl_quote(stfl_ipool_towc(ipool,text.c_str()))); stfl_ipool_destroy(ipool); return retval; diff --git a/src/utils.cpp b/src/utils.cpp index 92bc6413..904c4ca7 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -256,6 +256,11 @@ bool utils::try_fs_lock(const std::string& lock_file, pid_t & pid) { return false; } +std::string utils::translit(const char* tocode, const std::string& fromcode) +{ + return translit(std::string(tocode), fromcode); +} + std::string utils::translit(const std::string& tocode, const std::string& fromcode) { std::string tlit = "//TRANSLIT";