This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
from f7b7eff6541 [improvement](publish) Publish task not wait when be's
publish task a… (#28079)
add 2a05a52f979 [Feature](hive-writer) Implements s3 file committer.
(#33937)
No new revisions were added by this update.
Summary of changes:
be/src/io/fs/file_writer.h | 8 +
be/src/io/fs/s3_file_writer.cpp | 130 +++++----
be/src/io/fs/s3_file_writer.h | 18 ++
be/src/vec/sink/writer/vhive_partition_writer.cpp | 24 +-
be/src/vec/sink/writer/vhive_partition_writer.h | 1 +
be/src/vec/sink/writer/vhive_table_writer.cpp | 31 +-
.../doris/common/profile/SummaryProfile.java | 11 +
.../org/apache/doris/common/util/LocationPath.java | 13 +-
.../java/org/apache/doris/common/util/S3URI.java | 4 +-
.../doris/datasource/hive/HMSExternalCatalog.java | 14 +-
.../doris/datasource/hive/HMSTransaction.java | 263 ++++++++++++++---
.../doris/datasource/hive/HiveMetaStoreCache.java | 18 +-
.../doris/datasource/hive/HiveMetadataOps.java | 17 +-
.../doris/datasource/hudi/source/HudiScanNode.java | 2 +-
.../datasource/iceberg/source/IcebergScanNode.java | 4 +-
.../datasource/paimon/source/PaimonScanNode.java | 2 +-
.../main/java/org/apache/doris/fs/FileSystem.java | 8 +
.../java/org/apache/doris/fs/FileSystemCache.java | 17 +-
.../org/apache/doris/fs/FileSystemFactory.java | 5 +-
...FileSystemType.java => FileSystemProvider.java} | 11 +-
.../FileSystemProviderImpl.java} | 30 +-
.../org/apache/doris/fs/LocalDfsFileSystem.java | 5 +
.../org/apache/doris/fs/remote/ObjFileSystem.java | 15 +
.../org/apache/doris/fs/remote/S3FileSystem.java | 4 -
.../doris/fs/remote/SwitchingFileSystem.java | 132 +++++++++
.../org/apache/doris/planner/HiveTableSink.java | 37 ++-
.../doris/transaction/HiveTransactionManager.java | 13 +-
.../transaction/TransactionManagerFactory.java | 8 +-
.../apache/doris/common/util/LocationPathTest.java | 30 +-
.../doris/datasource/hive/HmsCommitTest.java | 32 ++-
gensrc/thrift/DataSinks.thrift | 10 +
.../hive/test_hive_write_insert_s3.out} | 320 +--------------------
.../hive/test_hive_write_insert_s3.groovy | 166 +++++++++++
33 files changed, 884 insertions(+), 519 deletions(-)
copy fe/fe-core/src/main/java/org/apache/doris/fs/{FileSystemType.java =>
FileSystemProvider.java} (86%)
copy
fe/fe-core/src/main/java/org/apache/doris/{datasource/hive/HiveDatabaseMetadata.java
=> fs/FileSystemProviderImpl.java} (53%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/fs/remote/SwitchingFileSystem.java
copy
regression-test/data/{external_table_p0/hive/write/test_hive_write_partitions.out
=> external_table_p2/hive/test_hive_write_insert_s3.out} (53%)
create mode 100644
regression-test/suites/external_table_p2/hive/test_hive_write_insert_s3.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
