This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4d52e23b4fc [feature](regression test & stream-load) Add the
enableUtf8Encoding field to control the behavior of the HTTP client encoding
header. (#56205)
4d52e23b4fc is described below
commit 4d52e23b4fc42b36d329d153e879f7aa6a610189
Author: Refrain <[email protected]>
AuthorDate: Fri Sep 19 10:13:31 2025 +0800
[feature](regression test & stream-load) Add the enableUtf8Encoding field
to control the behavior of the HTTP client encoding header. (#56205)
---
.../load_p0/stream_load/chinese_col_complex.csv | 3 +
.../regression/action/StreamLoadAction.groovy | 37 +++++-
.../stream_load/test_chinese_col_load.groovy | 128 +++++++++++++++++++++
3 files changed, 164 insertions(+), 4 deletions(-)
diff --git a/regression-test/data/load_p0/stream_load/chinese_col_complex.csv
b/regression-test/data/load_p0/stream_load/chinese_col_complex.csv
new file mode 100644
index 00000000000..62284cbf3bd
--- /dev/null
+++ b/regression-test/data/load_p0/stream_load/chinese_col_complex.csv
@@ -0,0 +1,3 @@
+1,FLOW001,张三
+2,FLOW002,王五
+3,FLOW003,孙七
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy
index 0916e91ee10..4d618c89b0f 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy
@@ -37,6 +37,10 @@ import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.CloseableHttpClient
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager
+import org.apache.http.config.ConnectionConfig
+import java.nio.charset.StandardCharsets
+import java.nio.charset.CodingErrorAction
import org.junit.Assert
import java.io.InputStream
import java.io.IOException
@@ -59,6 +63,7 @@ class StreamLoadAction implements SuiteAction {
SuiteContext context
boolean directToBe = false
boolean twoPhaseCommit = false
+ boolean enableUtf8Encoding = false // 新增：UTF-8编码支持开关
StreamLoadAction(SuiteContext context) {
this.address = context.getFeHttpAddress()
@@ -154,6 +159,14 @@ class StreamLoadAction implements SuiteAction {
this.twoPhaseCommit = twoPhaseCommit.call();
}
+ void enableUtf8Encoding(boolean enable) {
+ this.enableUtf8Encoding = enable
+ }
+
+ void enableUtf8Encoding(Closure<Boolean> enable) {
+ this.enableUtf8Encoding = enable.call()
+ }
+
// compatible with selectdb case
void isCloud(boolean isCloud) {
}
@@ -193,16 +206,32 @@ class StreamLoadAction implements SuiteAction {
} else {
uri =
"http://${address.hostString}:${address.port}/api/${db}/${table}/_stream_load";
}
- HttpClients.createDefault().withCloseable { client ->
+ def client
+ if (enableUtf8Encoding) {
+ // set UTF-8
+ def connConfig = ConnectionConfig.custom()
+ .setCharset(StandardCharsets.UTF_8)
+ .setMalformedInputAction(CodingErrorAction.REPLACE)
+ .setUnmappableInputAction(CodingErrorAction.REPLACE)
+ .build()
+ def cm = new PoolingHttpClientConnectionManager()
+ cm.setDefaultConnectionConfig(connConfig)
+ client = HttpClients.custom().setConnectionManager(cm).build()
+ log.info("Stream load using UTF-8 encoding for headers")
+ } else {
+ // default：ISO-8859-1
+ client = HttpClients.createDefault()
+ }
+ client.withCloseable { httpClient ->
RequestBuilder requestBuilder =
prepareRequestHeader(RequestBuilder.put(uri))
- HttpEntity httpEntity = prepareHttpEntity(client)
+ HttpEntity httpEntity = prepareHttpEntity(httpClient)
if (!directToBe) {
- String beLocation = streamLoadToFe(client, requestBuilder)
+ String beLocation = streamLoadToFe(httpClient,
requestBuilder)
log.info("Redirect stream load to
${beLocation}".toString())
requestBuilder.setUri(beLocation)
}
requestBuilder.setEntity(httpEntity)
- responseText = streamLoadToBe(client, requestBuilder)
+ responseText = streamLoadToBe(httpClient, requestBuilder)
}
} catch (Throwable t) {
ex = t
diff --git
a/regression-test/suites/load_p0/stream_load/test_chinese_col_load.groovy
b/regression-test/suites/load_p0/stream_load/test_chinese_col_load.groovy
new file mode 100644
index 00000000000..ccd08b9e4f4
--- /dev/null
+++ b/regression-test/suites/load_p0/stream_load/test_chinese_col_load.groovy
@@ -0,0 +1,128 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_chinese_col_load", "p0") {
+ def tableName = "test_chinese_col_load"
+
+ sql "SET enable_unicode_name_support = true"
+ sql "SET sql_mode = ''"
+
+ def stream_load_success = {table_name, _format, _columns, _jsonpaths,
col_sep,
+ array_json, line_json, file_name ->
+ streamLoad {
+ table table_name
+ // http request header
+ set 'format', _format
+ set 'columns', _columns
+ set 'jsonpaths', _jsonpaths
+ set 'column_separator', col_sep
+ set 'strip_outer_array', array_json
+ set 'read_json_by_line', line_json
+ set 'enable_unicode_name_support', 'true'
+ file file_name
+ time 10000 // limit inflight 10s
+
+ enableUtf8Encoding true
+
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ log.error("Unexpected exception with UTF-8 encoding:
${exception}")
+ throw exception
+ }
+ log.info("Stream load result (UTF-8 enabled):
${result}".toString())
+ def json = parseJson(result)
+
+ assertEquals("Success", json.Status)
+ assertEquals(3, json.NumberLoadedRows.toLong())
+ log.info("✅ Chinese column names handled correctly with UTF-8
encoding")
+ }
+ }
+ }
+
+ def stream_load_failure = {table_name, _format, _columns, _jsonpaths,
col_sep,
+ array_json, line_json, file_name ->
+ streamLoad {
+ table table_name
+ // http request header
+ set 'format', _format
+ set 'columns', _columns
+ set 'jsonpaths', _jsonpaths
+ set 'column_separator', col_sep
+ set 'strip_outer_array', array_json
+ set 'read_json_by_line', line_json
+ set 'enable_unicode_name_support', 'true'
+ file file_name
+ time 10000 // limit inflight 10s
+
+ enableUtf8Encoding false
+
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ log.info("Expected exception without UTF-8 encoding:
${exception}")
+ return
+ }
+ log.info("Stream load result (UTF-8 disabled):
${result}".toString())
+ def json = parseJson(result)
+
+ if ("Success".equals(json.Status)) {
+ log.warn("⚠️ Unexpected success without UTF-8 encoding -
server may have fallback handling")
+ log.info("Loaded rows: ${json.NumberLoadedRows}")
+ } else {
+ log.info("❌ Expected failure without UTF-8 encoding -
Status: ${json.Status}")
+ log.info("Error message: ${json.Message}")
+ if (json.ErrorURL) {
+ def (code, out, err) = curl("GET", json.ErrorURL)
+ log.info("Error details: " + out)
+ assertTrue("Should contain encoding-related error",
+ out.contains("Duplicate column") ||
out.contains("??") ||
+ json.Message.contains("ParseException") ||
+ json.Message.contains("Unknown column"))
+ }
+ }
+ }
+ }
+ }
+
+ def create_test_table = {table_name ->
+ sql """
+ CREATE TABLE IF NOT EXISTS ${table_name} (
+ `OBJECTID` varchar(36) NULL,
+ `姓名` varchar(255) NULL,
+ `CREATEDTIME` datetime NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`OBJECTID`)
+ DISTRIBUTED BY HASH(`OBJECTID`) BUCKETS 3
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ )
+ """
+ }
+
+ // case1: test UTF-8
+ sql "DROP TABLE IF EXISTS ${tableName}_success"
+ create_test_table.call("${tableName}_success")
+ log.info("🧪 Testing Chinese column names WITH UTF-8 encoding (should
succeed)")
+ stream_load_success.call("${tableName}_success", "csv",
"OBJECTID,姓名,CREATEDTIME", "", ",", "", "", "chinese_col_complex.csv")
+ try_sql("DROP TABLE IF EXISTS ${tableName}_success")
+
+ // case2: test default
+ sql "DROP TABLE IF EXISTS ${tableName}_failure"
+ create_test_table.call("${tableName}_failure")
+ log.info("🧪 Testing Chinese column names WITHOUT UTF-8 encoding (may
fail)")
+ stream_load_failure.call("${tableName}_failure", "csv",
"OBJECTID,姓名,CREATEDTIME", "", ",", "", "", "chinese_col_complex.csv")
+ try_sql("DROP TABLE IF EXISTS ${tableName}_failure")
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
