This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch dev-1.1.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/dev-1.1.2 by this push:
new 54f4a8be57 [Feature] (ODBC) support read/write emoji of utf16 via odbc
table (#11863)
54f4a8be57 is described below
commit 54f4a8be578c024d4161ea42fa9640d3b3124087
Author: HappenLee <[email protected]>
AuthorDate: Thu Aug 18 09:09:02 2022 +0800
[Feature] (ODBC) support read/write emoji of utf16 via odbc table (#11863)
Co-authored-by: lihaopeng <[email protected]>
---
be/src/exec/odbc_connector.cpp | 26 +++++++---------------
.../java/org/apache/doris/catalog/Catalog.java | 1 +
2 files changed, 9 insertions(+), 18 deletions(-)
diff --git a/be/src/exec/odbc_connector.cpp b/be/src/exec/odbc_connector.cpp
index 1de8337e5d..1a12884cb0 100644
--- a/be/src/exec/odbc_connector.cpp
+++ b/be/src/exec/odbc_connector.cpp
@@ -48,14 +48,9 @@ static constexpr uint32_t BIG_COLUMN_SIZE_BUFFER = 65535;
// Default max buffer size use in insert to: 50MB, normally a batch is smaller
than the size
static constexpr uint32_t INSERT_BUFFER_SIZE = 1024l * 1024 * 50;
-static doris::Status utf8_to_wstring(const std::string& str, std::u16string&
out) {
- std::wstring_convert<std::codecvt_utf8<char16_t>, char16_t> utf8_ucs2_cvt;
- try {
- out = utf8_ucs2_cvt.from_bytes(str);
- } catch (std::range_error& e) {
- return doris::Status::InternalError("UNICODE out of supported range");
- }
- return doris::Status::OK();
+static std::u16string utf8_to_u16string(const char* first, const char* last) {
+ std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t>
utf8_utf16_cvt;
+ return utf8_utf16_cvt.from_bytes(first, last);
}
namespace doris {
@@ -133,8 +128,7 @@ Status ODBCConnector::query() {
"alloc statement");
// Translate utf8 string to utf16 to use unicode encoding
- std::u16string wquery;
- RETURN_IF_ERROR(utf8_to_wstring(_sql_str, wquery));
+ auto wquery = utf8_to_u16string(_sql_str.c_str(), _sql_str.c_str() +
_sql_str.length());
ODBC_DISPOSE(_stmt, SQL_HANDLE_STMT,
SQLExecDirectW(_stmt, (SQLWCHAR*)(wquery.c_str()), SQL_NTS),
"exec direct");
@@ -315,10 +309,8 @@ Status ODBCConnector::append(const std::string&
table_name, RowBatch* batch,
}
}
// Translate utf8 string to utf16 to use unicode encodeing
- RETURN_IF_ERROR(utf8_to_wstring(
- std::string(_insert_stmt_buffer.data(),
- _insert_stmt_buffer.data() +
_insert_stmt_buffer.size()),
- insert_stmt));
+ insert_stmt = utf8_to_u16string(_insert_stmt_buffer.data(),
+ _insert_stmt_buffer.data() +
_insert_stmt_buffer.size());
}
{
@@ -501,10 +493,8 @@ Status ODBCConnector::append(const std::string&
table_name, vectorized::Block* b
}
}
// Translate utf8 string to utf16 to use unicode encodeing
- RETURN_IF_ERROR(utf8_to_wstring(
- std::string(_insert_stmt_buffer.data(),
- _insert_stmt_buffer.data() +
_insert_stmt_buffer.size()),
- insert_stmt));
+ insert_stmt = utf8_to_u16string(_insert_stmt_buffer.data(),
+ _insert_stmt_buffer.data() +
_insert_stmt_buffer.size());
}
{
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
index ffcc3c8172..e5eb9c9232 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
@@ -4330,6 +4330,7 @@ public class Catalog {
sb.append("\"password\" = \"").append(hidePassword ? "" :
odbcTable.getPasswd()).append("\",\n");
sb.append("\"driver\" =
\"").append(odbcTable.getOdbcDriver()).append("\",\n");
sb.append("\"odbc_type\" =
\"").append(odbcTable.getOdbcTableTypeName()).append("\",\n");
+ sb.append("\"charest\" =
\"").append(odbcTable.getCharset()).append("\",\n");
} else {
sb.append("\"odbc_catalog_resource\" =
\"").append(odbcTable.getOdbcCatalogResourceName()).append("\",\n");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
