This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 5b8421f1aad [fix](outfile) fix outfile csv did not write json column
with string (#29067)
5b8421f1aad is described below
commit 5b8421f1aad421475756faf6232653f467279759
Author: xueweizhang <[email protected]>
AuthorDate: Thu Feb 1 18:47:30 2024 +0800
[fix](outfile) fix outfile csv did not write json column with string
(#29067)
---
.../vec/data_types/serde/data_type_jsonb_serde.cpp | 6 +++-
.../data/export_p0/test_outfile_csv_with_names.out | 40 +++++++++++-----------
.../export_p0/test_outfile_csv_with_names.groovy | 17 +++++----
3 files changed, 35 insertions(+), 28 deletions(-)
diff --git a/be/src/vec/data_types/serde/data_type_jsonb_serde.cpp
b/be/src/vec/data_types/serde/data_type_jsonb_serde.cpp
index 8a6b26e9bd8..2015ed082b1 100644
--- a/be/src/vec/data_types/serde/data_type_jsonb_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_jsonb_serde.cpp
@@ -83,7 +83,11 @@ Status DataTypeJsonbSerDe::serialize_one_cell_to_json(const
IColumn& column, int
const StringRef& s = assert_cast<const
ColumnString&>(*ptr).get_data_at(row_num);
if (s.size > 0) {
- bw.write(s.data, s.size);
+ std::string str = JsonbToJson::jsonb_to_json_string(s.data, s.size);
+ bw.write(str.c_str(), str.size());
+ } else {
+ bw.write(NULL_IN_CSV_FOR_ORDINARY_TYPE.c_str(),
+ strlen(NULL_IN_CSV_FOR_ORDINARY_TYPE.c_str()));
}
return Status::OK();
}
diff --git a/regression-test/data/export_p0/test_outfile_csv_with_names.out
b/regression-test/data/export_p0/test_outfile_csv_with_names.out
index 63fa1d523c8..5a6badd50e2 100644
--- a/regression-test/data/export_p0/test_outfile_csv_with_names.out
+++ b/regression-test/data/export_p0/test_outfile_csv_with_names.out
@@ -1,25 +1,25 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
-- !select_default1 --
-1 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 1 1 true 1 1 1 1.1 1.1 char1
1
-10 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
\N \N \N \N \N \N \N \N \N \N
\N
-2 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 2 2 true 2 2 2 2.2 2.2 char2
2
-3 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 3 3 true 3 3 3 3.3 3.3 char3
3
-4 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 4 4 true 4 4 4 4.4 4.4 char4
4
-5 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 5 5 true 5 5 5 5.5 5.5 char5
5
-6 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 6 6 true 6 6 6 6.6 6.6 char6
6
-7 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 7 7 true 7 7 7 7.7 7.7 char7
7
-8 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 8 8 true 8 8 8 8.8 8.8 char8
8
-9 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 9 9 true 9 9 9 9.9 9.9 char9
9
+1 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 1 1 true 1 1 1 1.1 1.1 char1
1 {"a":1,"b":"str1"}
+10 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
\N \N \N \N \N \N \N \N \N \N
\N \N
+2 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 2 2 true 2 2 2 2.2 2.2 char2
2 {"a":2,"b":"str2"}
+3 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 3 3 true 3 3 3 3.3 3.3 char3
3 {"a":3,"b":"str3"}
+4 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 4 4 true 4 4 4 4.4 4.4 char4
4 {"a":4,"b":"str4"}
+5 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 5 5 true 5 5 5 5.5 5.5 char5
5 {"a":5,"b":"str5"}
+6 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 6 6 true 6 6 6 6.6 6.6 char6
6 {"a":6,"b":"str6"}
+7 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 7 7 true 7 7 7 7.7 7.7 char7
7 {"a":7,"b":"str7"}
+8 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 8 8 true 8 8 8 8.8 8.8 char8
8 {"a":8,"b":"str8"}
+9 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 9 9 true 9 9 9 9.9 9.9 char9
9 {"a":9,"b":"str9"}
-- !select_default2 --
-1 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 1 1 true 1 1 1 1.1 1.1 char1
1
-10 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
\N \N \N \N \N \N \N \N \N \N
\N
-2 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 2 2 true 2 2 2 2.2 2.2 char2
2
-3 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 3 3 true 3 3 3 3.3 3.3 char3
3
-4 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 4 4 true 4 4 4 4.4 4.4 char4
4
-5 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 5 5 true 5 5 5 5.5 5.5 char5
5
-6 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 6 6 true 6 6 6 6.6 6.6 char6
6
-7 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 7 7 true 7 7 7 7.7 7.7 char7
7
-8 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 8 8 true 8 8 8 8.8 8.8 char8
8
-9 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 9 9 true 9 9 9 9.9 9.9 char9
9
+1 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 1 1 true 1 1 1 1.1 1.1 char1
1 {"a":1,"b":"str1"}
+10 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
\N \N \N \N \N \N \N \N \N \N
\N \N
+2 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 2 2 true 2 2 2 2.2 2.2 char2
2 {"a":2,"b":"str2"}
+3 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 3 3 true 3 3 3 3.3 3.3 char3
3 {"a":3,"b":"str3"}
+4 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 4 4 true 4 4 4 4.4 4.4 char4
4 {"a":4,"b":"str4"}
+5 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 5 5 true 5 5 5 5.5 5.5 char5
5 {"a":5,"b":"str5"}
+6 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 6 6 true 6 6 6 6.6 6.6 char6
6 {"a":6,"b":"str6"}
+7 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 7 7 true 7 7 7 7.7 7.7 char7
7 {"a":7,"b":"str7"}
+8 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 8 8 true 8 8 8 8.8 8.8 char8
8 {"a":8,"b":"str8"}
+9 2017-10-01 2017-10-01T00:00 2017-10-01
2017-10-01T00:00 2017-10-01T00:00:00.111 2017-10-01T00:00:00.111111
Beijing 9 9 true 9 9 9 9.9 9.9 char9
9 {"a":9,"b":"str9"}
diff --git
a/regression-test/suites/export_p0/test_outfile_csv_with_names.groovy
b/regression-test/suites/export_p0/test_outfile_csv_with_names.groovy
index 9c62a994c47..97722100191 100644
--- a/regression-test/suites/export_p0/test_outfile_csv_with_names.groovy
+++ b/regression-test/suites/export_p0/test_outfile_csv_with_names.groovy
@@ -77,7 +77,8 @@ suite("test_outfile_csv_with_names") {
`float_col` float COMMENT "",
`double_col` double COMMENT "",
`char_col` CHAR(10) COMMENT "",
- `decimal_col` decimal COMMENT ""
+ `decimal_col` decimal COMMENT "",
+ `json_col` json COMMENT ""
)
DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
"""
@@ -85,11 +86,11 @@ suite("test_outfile_csv_with_names") {
int i = 1
for (; i < 10; i ++) {
sb.append("""
- (${i}, '2017-10-01', '2017-10-01 00:00:00', '2017-10-01',
'2017-10-01 00:00:00.111111', '2017-10-01 00:00:00.111111', '2017-10-01
00:00:00.111111', 'Beijing', ${i}, ${i % 128}, true, ${i}, ${i}, ${i},
${i}.${i}, ${i}.${i}, 'char${i}', ${i}),
+ (${i}, '2017-10-01', '2017-10-01 00:00:00', '2017-10-01',
'2017-10-01 00:00:00.111111', '2017-10-01 00:00:00.111111', '2017-10-01
00:00:00.111111', 'Beijing', ${i}, ${i % 128}, true, ${i}, ${i}, ${i},
${i}.${i}, ${i}.${i}, 'char${i}', ${i}, '{"a": ${i}, "b": "str${i}"}'),
""")
}
sb.append("""
- (${i}, '2017-10-01', '2017-10-01 00:00:00', '2017-10-01',
'2017-10-01 00:00:00.111111', '2017-10-01 00:00:00.111111', '2017-10-01
00:00:00.111111', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL)
+ (${i}, '2017-10-01', '2017-10-01 00:00:00', '2017-10-01',
'2017-10-01 00:00:00.111111', '2017-10-01 00:00:00.111111', '2017-10-01
00:00:00.111111', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL)
""")
sql """ INSERT INTO ${tableName} VALUES
${sb.toString()}
@@ -105,13 +106,13 @@ suite("test_outfile_csv_with_names") {
}
sql """
SELECT * FROM ${tableName} t ORDER BY user_id INTO OUTFILE
"file://${outFilePath}/" FORMAT AS CSV_WITH_NAMES
- PROPERTIES("column_separator" = ",");
+ PROPERTIES("column_separator" = "|");
"""
File[] files = path.listFiles()
assert files.length == 1
// check column names
- String columnNames =
"""user_id,date,datetime,date_1,datetime_1,datetime_2,datetime_3,city,age,sex,bool_col,int_col,bigint_col,largeint_col,float_col,double_col,char_col,decimal_col"""
+ String columnNames =
"""user_id|date|datetime|date_1|datetime_1|datetime_2|datetime_3|city|age|sex|bool_col|int_col|bigint_col|largeint_col|float_col|double_col|char_col|decimal_col|json_col"""
List<String> outLines =
Files.readAllLines(Paths.get(files[0].getAbsolutePath()),
StandardCharsets.UTF_8);
assertEquals(columnNames, outLines.get(0))
@@ -137,14 +138,15 @@ suite("test_outfile_csv_with_names") {
`float_col` float COMMENT "",
`double_col` double COMMENT "",
`char_col` CHAR(10) COMMENT "",
- `decimal_col` decimal COMMENT ""
+ `decimal_col` decimal COMMENT "",
+ `json_col` json COMMENT ""
)
DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
"""
StringBuilder commandBuilder = new StringBuilder()
commandBuilder.append("""curl -v --location-trusted -u
${context.config.feHttpUser}:${context.config.feHttpPassword}""")
- commandBuilder.append(""" -H format:csv_with_names -H
column_separator:, -T """ + files[0].getAbsolutePath() + """
http://${context.config.feHttpAddress}/api/"""; + dbName + "/" + tableName2 +
"/_stream_load")
+ commandBuilder.append(""" -H format:csv_with_names -H
column_separator:| -T """ + files[0].getAbsolutePath() + """
http://${context.config.feHttpAddress}/api/"""; + dbName + "/" + tableName2 +
"/_stream_load")
command = commandBuilder.toString()
process = command.execute()
code = process.waitFor()
@@ -165,3 +167,4 @@ suite("test_outfile_csv_with_names") {
}
}
}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
