This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 96ed07c83ef [fix](test) fix some p2 external table test cases (#33624)
96ed07c83ef is described below
commit 96ed07c83ef116df481edefd0a5001b684931260
Author: Mingyu Chen <morning...@163.com>
AuthorDate: Sat Apr 13 23:26:54 2024 +0800
[fix](test) fix some p2 external table test cases (#33624)
bp #33621
Also fix a merge bug from #33245
---
be/src/vec/sink/writer/vhive_partition_writer.cpp | 14 +--
be/src/vec/sink/writer/vhive_partition_writer.h | 3 +
.../java/org/apache/doris/qe/ConnectContext.java | 2 +-
.../hive/test_hive_parquet_alter_column.out | 114 ++++++++++-----------
...est_external_catalog_iceberg_hadoop_catalog.out | 9 +-
.../test_external_catalog_iceberg_partition.out | 46 ++++-----
.../hive/ddl/test_hive_ddl.groovy | 1 +
.../hive/ddl/test_hive_write_type.groovy | 1 +
8 files changed, 99 insertions(+), 91 deletions(-)
diff --git a/be/src/vec/sink/writer/vhive_partition_writer.cpp
b/be/src/vec/sink/writer/vhive_partition_writer.cpp
index f8fd13c5a27..a67a22e2cf1 100644
--- a/be/src/vec/sink/writer/vhive_partition_writer.cpp
+++ b/be/src/vec/sink/writer/vhive_partition_writer.cpp
@@ -56,9 +56,8 @@ Status VHivePartitionWriter::open(RuntimeState* state,
RuntimeProfile* profile)
std::vector<TNetworkAddress> broker_addresses;
RETURN_IF_ERROR(FileFactory::create_file_writer(
_write_info.file_type, state->exec_env(), broker_addresses,
_hadoop_conf,
- fmt::format("{}/{}", _write_info.write_path, _file_name,
_file_name_index,
- _get_file_extension(_file_format_type,
_hive_compress_type)),
- 0, _file_writer));
+ fmt::format("{}/{}", _write_info.write_path,
_get_target_file_name()), 0,
+ _file_writer));
std::vector<std::string> column_names;
column_names.reserve(_columns.size());
@@ -195,9 +194,7 @@ THivePartitionUpdate
VHivePartitionWriter::_build_partition_update() {
location.__set_write_path(_write_info.write_path);
location.__set_target_path(_write_info.target_path);
hive_partition_update.__set_location(location);
- hive_partition_update.__set_file_names(
- {fmt::format("{}-{}{}", _file_name, _file_name_index,
- _get_file_extension(_file_format_type,
_hive_compress_type))});
+ hive_partition_update.__set_file_names({_get_target_file_name()});
hive_partition_update.__set_row_count(_row_count);
hive_partition_update.__set_file_size(_input_size_in_bytes);
return hive_partition_update;
@@ -243,5 +240,10 @@ std::string
VHivePartitionWriter::_get_file_extension(TFileFormatType::type file
return fmt::format("{}{}", compress_name, file_format_name);
}
+std::string VHivePartitionWriter::_get_target_file_name() {
+ return fmt::format("{}-{}{}", _file_name, _file_name_index,
+ _get_file_extension(_file_format_type,
_hive_compress_type));
+}
+
} // namespace vectorized
} // namespace doris
diff --git a/be/src/vec/sink/writer/vhive_partition_writer.h
b/be/src/vec/sink/writer/vhive_partition_writer.h
index b8735ddcf7d..fc06e157408 100644
--- a/be/src/vec/sink/writer/vhive_partition_writer.h
+++ b/be/src/vec/sink/writer/vhive_partition_writer.h
@@ -67,6 +67,9 @@ public:
inline size_t written_len() { return
_file_format_transformer->written_len(); }
+private:
+ std::string _get_target_file_name();
+
private:
Status _projection_and_filter_block(doris::vectorized::Block& input_block,
const vectorized::IColumn::Filter*
filter,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
index d561494fe52..dabdf6b3346 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
@@ -575,7 +575,7 @@ public class ConnectContext {
// for USER() function
public UserIdentity getUserIdentity() {
- return new UserIdentity(qualifiedUser, remoteIP);
+ return UserIdentity.createAnalyzedUserIdentWithIp(qualifiedUser,
remoteIP);
}
public UserIdentity getCurrentUserIdentity() {
diff --git
a/regression-test/data/external_table_p2/hive/test_hive_parquet_alter_column.out
b/regression-test/data/external_table_p2/hive/test_hive_parquet_alter_column.out
index 5c0d1228d5a..d566c008c0d 100644
---
a/regression-test/data/external_table_p2/hive/test_hive_parquet_alter_column.out
+++
b/regression-test/data/external_table_p2/hive/test_hive_parquet_alter_column.out
@@ -638,13 +638,13 @@ col_timestamp TEXT Yes true \N
col_decimal TEXT Yes true \N
-- !show --
--1 -200 -10 -20000000 2.05777E1 3.075E1 false First
A ADC 2023-10-06�� 2023-10-09 17:15:00�� 1238.45
--1 -200 -10 -20000000 2.05777E1 3.075E1 false First
A ADC 2023-10-06�� 2023-10-09 17:15:00�� 1238.45
--1 -200 -10 -20000000 2.05777E1 3.075E1 false First
A ADC 2023-10-06�� 2023-10-09 17:15:00�� 1238.45
--1 -200 -10 -20000000 2.05777E1 3.075E1 false First
A ADC 2023-10-06�� 2023-10-09 17:15:00�� 1238.45
--1 -200 -10 -20000000 2.05777E1 3.075E1 false First
A ADC 2023-10-06�� 2023-10-09 17:15:00�� 1238.45
--1 -200 -10 -20000000 2.05777E1 3.075E1 false First
A ADC 2023-10-06�� 2023-10-09 17:15:00�� 1238.45
--1 -200 -10 -20000000 2.05777E1 3.075E1 false First
A ADC 2023-10-06�� 2023-10-09 17:15:00�� 1238.45
+-1 -200 -10 -20000000 20.577700 30.750000 false
First A ADC 2023-10-06 2023-10-09 17:15:00 1238.45
+-1 -200 -10 -20000000 20.577700 30.750000 false
First A ADC 2023-10-06 2023-10-09 17:15:00 1238.45
+-1 -200 -10 -20000000 20.577700 30.750000 false
First A ADC 2023-10-06 2023-10-09 17:15:00 1238.45
+-1 -200 -10 -20000000 20.577700 30.750000 false
First A ADC 2023-10-06 2023-10-09 17:15:00 1238.45
+-1 -200 -10 -20000000 20.577700 30.750000 false
First A ADC 2023-10-06 2023-10-09 17:15:00 1238.45
+-1 -200 -10 -20000000 20.577700 30.750000 false
First A ADC 2023-10-06 2023-10-09 17:15:00 1238.45
+-1 -200 -10 -20000000 20.577700 30.750000 false
First A ADC 2023-10-06 2023-10-09 17:15:00 1238.45
-- !order --
-1
@@ -667,14 +667,14 @@ col_decimal TEXT Yes true \N
-20000000
-- !order --
-1.05E1
-1.05E1
-1.05E1
+10.500000
+10.500000
+10.500000
-- !order --
-2.075E1
-2.075E1
-2.075E1
+20.750000
+20.750000
+20.750000
-- !order --
false
@@ -697,14 +697,14 @@ ADC
ADC
-- !order --
-2023-10-06��
-2023-10-06��
-2023-10-06��
+2023-10-06
+2023-10-06
+2023-10-06
-- !order --
-2023-10-06 14:30:00��
-2023-10-06 14:30:00��
-2023-10-06 14:30:00��
+2023-10-06 14:30:00
+2023-10-06 14:30:00
+2023-10-06 14:30:00
-- !order --
123.45
@@ -727,13 +727,13 @@ col_timestamp CHAR(10) Yes true \N
col_decimal CHAR(10) Yes true \N
-- !show --
--1 -200 -10 -20000000 2.05777E1 3.075E1 false First
A ADC 2023-10-06�� 2023-10-09 17:15:00�� 1238.45
--1 -200 -10 -20000000 2.05777E1 3.075E1 false First
A ADC 2023-10-06�� 2023-10-09 17:15:00�� 1238.45
--1 -200 -10 -20000000 2.05777E1 3.075E1 false First
A ADC 2023-10-06�� 2023-10-09 17:15:00�� 1238.45
--1 -200 -10 -20000000 2.05777E1 3.075E1 false First
A ADC 2023-10-06�� 2023-10-09 17:15:00�� 1238.45
--1 -200 -10 -20000000 2.05777E1 3.075E1 false First
A ADC 2023-10-06�� 2023-10-09 17:15:00�� 1238.45
--1 -200 -10 -20000000 2.05777E1 3.075E1 false First
A ADC 2023-10-06�� 2023-10-09 17:15:00�� 1238.45
--1 -200 -10 -20000000 2.05777E1 3.075E1 false First
A ADC 2023-10-06�� 2023-10-09 17:15:00�� 1238.45
+-1 -200 -10 -20000000 20.577700 30.750000 false
First A ADC 2023-10-06 2023-10-09 17:15:00 1238.45
+-1 -200 -10 -20000000 20.577700 30.750000 false
First A ADC 2023-10-06 2023-10-09 17:15:00 1238.45
+-1 -200 -10 -20000000 20.577700 30.750000 false
First A ADC 2023-10-06 2023-10-09 17:15:00 1238.45
+-1 -200 -10 -20000000 20.577700 30.750000 false
First A ADC 2023-10-06 2023-10-09 17:15:00 1238.45
+-1 -200 -10 -20000000 20.577700 30.750000 false
First A ADC 2023-10-06 2023-10-09 17:15:00 1238.45
+-1 -200 -10 -20000000 20.577700 30.750000 false
First A ADC 2023-10-06 2023-10-09 17:15:00 1238.45
+-1 -200 -10 -20000000 20.577700 30.750000 false
First A ADC 2023-10-06 2023-10-09 17:15:00 1238.45
-- !order --
-1
@@ -756,14 +756,14 @@ col_decimal CHAR(10) Yes true \N
-20000000
-- !order --
-1.05E1
-1.05E1
-1.05E1
+10.500000
+10.500000
+10.500000
-- !order --
-2.075E1
-2.075E1
-2.075E1
+20.750000
+20.750000
+20.750000
-- !order --
false
@@ -786,14 +786,14 @@ ADC
ADC
-- !order --
-2023-10-06��
-2023-10-06��
-2023-10-06��
+2023-10-06
+2023-10-06
+2023-10-06
-- !order --
-2023-10-06 14:30:00��
-2023-10-06 14:30:00��
-2023-10-06 14:30:00��
+2023-10-06 14:30:00
+2023-10-06 14:30:00
+2023-10-06 14:30:00
-- !order --
123.45
@@ -816,13 +816,13 @@ col_timestamp VARCHAR(20) Yes true \N
col_decimal VARCHAR(20) Yes true \N
-- !show --
--1 -200 -10 -20000000 2.05777E1 3.075E1 false First
A ADC 2023-10-06�� 2023-10-09 17:15:00�� 1238.45
--1 -200 -10 -20000000 2.05777E1 3.075E1 false First
A ADC 2023-10-06�� 2023-10-09 17:15:00�� 1238.45
--1 -200 -10 -20000000 2.05777E1 3.075E1 false First
A ADC 2023-10-06�� 2023-10-09 17:15:00�� 1238.45
--1 -200 -10 -20000000 2.05777E1 3.075E1 false First
A ADC 2023-10-06�� 2023-10-09 17:15:00�� 1238.45
--1 -200 -10 -20000000 2.05777E1 3.075E1 false First
A ADC 2023-10-06�� 2023-10-09 17:15:00�� 1238.45
--1 -200 -10 -20000000 2.05777E1 3.075E1 false First
A ADC 2023-10-06�� 2023-10-09 17:15:00�� 1238.45
--1 -200 -10 -20000000 2.05777E1 3.075E1 false First
A ADC 2023-10-06�� 2023-10-09 17:15:00�� 1238.45
+-1 -200 -10 -20000000 20.577700 30.750000 false
First A ADC 2023-10-06 2023-10-09 17:15:00 1238.45
+-1 -200 -10 -20000000 20.577700 30.750000 false
First A ADC 2023-10-06 2023-10-09 17:15:00 1238.45
+-1 -200 -10 -20000000 20.577700 30.750000 false
First A ADC 2023-10-06 2023-10-09 17:15:00 1238.45
+-1 -200 -10 -20000000 20.577700 30.750000 false
First A ADC 2023-10-06 2023-10-09 17:15:00 1238.45
+-1 -200 -10 -20000000 20.577700 30.750000 false
First A ADC 2023-10-06 2023-10-09 17:15:00 1238.45
+-1 -200 -10 -20000000 20.577700 30.750000 false
First A ADC 2023-10-06 2023-10-09 17:15:00 1238.45
+-1 -200 -10 -20000000 20.577700 30.750000 false
First A ADC 2023-10-06 2023-10-09 17:15:00 1238.45
-- !order --
-1
@@ -845,14 +845,14 @@ col_decimal VARCHAR(20) Yes true \N
-20000000
-- !order --
-1.05E1
-1.05E1
-1.05E1
+10.500000
+10.500000
+10.500000
-- !order --
-2.075E1
-2.075E1
-2.075E1
+20.750000
+20.750000
+20.750000
-- !order --
false
@@ -875,14 +875,14 @@ ADC
ADC
-- !order --
-2023-10-06��
-2023-10-06��
-2023-10-06��
+2023-10-06
+2023-10-06
+2023-10-06
-- !order --
-2023-10-06 14:30:00��
-2023-10-06 14:30:00��
-2023-10-06 14:30:00��
+2023-10-06 14:30:00
+2023-10-06 14:30:00
+2023-10-06 14:30:00
-- !order --
123.45
diff --git
a/regression-test/data/external_table_p2/iceberg/test_external_catalog_iceberg_hadoop_catalog.out
b/regression-test/data/external_table_p2/iceberg/test_external_catalog_iceberg_hadoop_catalog.out
index aaa9037977d..c9373c2df17 100644
---
a/regression-test/data/external_table_p2/iceberg/test_external_catalog_iceberg_hadoop_catalog.out
+++
b/regression-test/data/external_table_p2/iceberg/test_external_catalog_iceberg_hadoop_catalog.out
@@ -17,10 +17,11 @@
5 Customer#000000005 hwBtxkoBF qSW4KrI CANADA 5 CANADA
AMERICA 13-750-942-6364 HOUSEHOLD
-- !q04 --
-1 1970-01-03 09:02:03.000001 a
-1 1970-01-03 09:02:03.000001 b
-2 1970-01-03 09:02:04.000001 c
-2 1970-01-03 09:02:04.000001 d
+1 1970-01-03T01:02:03.000001 a
+1 1970-01-03T01:02:03.000001 b
+2 1970-01-03T01:02:04.000001 c
+2 1970-01-03T01:02:04.000001 d
-- !q05 --
463870
+
diff --git
a/regression-test/data/external_table_p2/iceberg/test_external_catalog_iceberg_partition.out
b/regression-test/data/external_table_p2/iceberg/test_external_catalog_iceberg_partition.out
index a0fd39be5f2..14212c9e7af 100644
---
a/regression-test/data/external_table_p2/iceberg/test_external_catalog_iceberg_partition.out
+++
b/regression-test/data/external_table_p2/iceberg/test_external_catalog_iceberg_partition.out
@@ -1,39 +1,39 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
-- !q01 --
-\N 0.1 test1 2023-01-01T00:00
-\N 0.2 test2 2023-01-02T00:00
-100 0.3 test3 2023-01-03T00:00
+\N 0.1 test1 2022-12-31T16:00
+\N 0.2 test2 2023-01-01T16:00
+100 0.3 test3 2023-01-02T16:00
-- !q02 --
-\N 0.1 test1 2023-01-01T00:00
-\N 0.2 test2 2023-01-02T00:00
+\N 0.1 test1 2022-12-31T16:00
+\N 0.2 test2 2023-01-01T16:00
-- !q03 --
-100 0.3 test3 2023-01-03T00:00
+100 0.3 test3 2023-01-02T16:00
-- !q04 --
-\N 0.1 test1 2023-01-01T00:00
-\N 0.2 test2 2023-01-02T00:00
-100 0.3 test3 2023-01-03T00:00
+\N 0.1 test1 2022-12-31T16:00
+\N 0.2 test2 2023-01-01T16:00
+100 0.3 test3 2023-01-02T16:00
-- !q05 --
-\N 0.1 test1 2023-01-01T00:00
-\N 0.2 test2 2023-01-02T00:00
+\N 0.1 test1 2022-12-31T16:00
+\N 0.2 test2 2023-01-01T16:00
-- !q06 --
-100 0.3 test3 2023-01-03T00:00
+100 0.3 test3 2023-01-02T16:00
-- !q07 --
-\N 0.1 test1 2023-01-01T00:00
-\N 0.2 test2 2023-01-02T00:00
-100 0.3 test3 2023-01-03T00:00
+\N 0.1 test1 2022-12-31T16:00
+\N 0.2 test2 2023-01-01T16:00
+100 0.3 test3 2023-01-02T16:00
-- !q08 --
-\N 0.1 test1 2023-01-01T00:00
-\N 0.2 test2 2023-01-02T00:00
+\N 0.1 test1 2022-12-31T16:00
+\N 0.2 test2 2023-01-01T16:00
-- !q09 --
-100 0.3 test3 2023-01-03T00:00
+100 0.3 test3 2023-01-02T16:00
-- !q01 --
\N 0.1 test1 2023-01-01T00:00
@@ -72,19 +72,19 @@
100 0.3 test3 2023-01-03T00:00
-- !q01 --
-1 true 2020-01-02 2020-01-02T01:02:03.123456
+1 true 2020-01-02 2020-01-01T17:02:03.123456
-- !q02 --
-1 true 2020-01-02 2020-01-02T01:02:03.123456
+1 true 2020-01-02 2020-01-01T17:02:03.123456
-- !q03 --
-1 true 2020-01-02 2020-01-02T01:02:03.123456
+1 true 2020-01-02 2020-01-01T17:02:03.123456
-- !q04 --
-1 true 2020-01-02 2020-01-02T01:02:03.123456
+1 true 2020-01-02 2020-01-01T17:02:03.123456
-- !q05 --
-1 true 2020-01-02 2020-01-02T01:02:03.123456
+1 true 2020-01-02 2020-01-01T17:02:03.123456
-- !q06 --
1 2020-01-02
diff --git
a/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy
b/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy
index b0543931371..41e7658c17c 100644
--- a/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy
+++ b/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy
@@ -419,6 +419,7 @@ suite("test_hive_ddl",
"p0,external,hive,external_docker,external_docker_hive")
sql """ create database if not exists `test_hive_db_tbl` """;
sql """use `${catalog_name}`.`test_hive_db_tbl`"""
+ sql """ drop table if exists unpart_tbl_${file_format}"""
sql """
CREATE TABLE unpart_tbl_${file_format}(
`col1` BOOLEAN COMMENT 'col1',
diff --git
a/regression-test/suites/external_table_p0/hive/ddl/test_hive_write_type.groovy
b/regression-test/suites/external_table_p0/hive/ddl/test_hive_write_type.groovy
index 07953ed1575..41be3ceded6 100644
---
a/regression-test/suites/external_table_p0/hive/ddl/test_hive_write_type.groovy
+++
b/regression-test/suites/external_table_p0/hive/ddl/test_hive_write_type.groovy
@@ -24,6 +24,7 @@ suite("test_hive_write_type",
"p0,external,hive,external_docker,external_docker_
sql """ create database if not exists `test_complex_type` """;
sql """ use `${catalog_name}`.`test_complex_type` """
+ sql """ drop table if exists unpart_tbl_${file_format} """
sql """
CREATE TABLE unpart_tbl_${file_format} (
`col1` CHAR,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
