This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
from e3651b64b9 [fix](regression case) fix variable scope bug in some
inverted index regression cases (#21194)
new 2a14fca2e7 [feature](load-refactor-with-tvf) S3 load with S3 tvf and
native insert (#19937)
new e8d48413c1 [Feature](avro) Support Apache Avro file format (#19990)
new cf9969af31 [opt](hudi) use spark bundle to read hudi data (#21260)
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/service/internal_service.cpp | 9 +
be/src/vec/exec/jni_connector.cpp | 21 +-
be/src/vec/exec/jni_connector.h | 24 +
be/src/vec/exec/scan/avro_jni_reader.cpp | 165 +++++
.../scan/{hudi_jni_reader.h => avro_jni_reader.h} | 41 +-
be/src/vec/exec/scan/hudi_jni_reader.cpp | 6 +-
be/src/vec/exec/scan/hudi_jni_reader.h | 2 +-
be/src/vec/exec/scan/vfile_scanner.cpp | 7 +
bin/start_be.sh | 12 +-
build.sh | 2 +
.../sql-manual/sql-functions/table-functions/s3.md | 14 +-
.../sql-manual/sql-functions/table-functions/s3.md | 10 +-
.../{hudi-scanner => avro-scanner}/pom.xml | 87 +--
.../org/apache/doris/avro/AvroColumnValue.java} | 103 ++-
.../java/org/apache/doris/avro/AvroJNIScanner.java | 248 ++++++++
.../java/org/apache/doris/avro/AvroProperties.java | 40 ++
.../java/org/apache/doris/avro/AvroReader.java | 37 ++
.../java/org/apache/doris/avro/HDFSFileReader.java | 73 +++
.../java/org/apache/doris/avro/S3FileReader.java | 91 +++
.../src/main/resources/package.xml | 0
fe/be-java-extensions/hudi-scanner/pom.xml | 201 +++++-
.../org/apache/doris/hudi/HudiColumnValue.java | 141 +++--
.../java/org/apache/doris/hudi/HudiJniScanner.java | 188 ++----
.../java/org/apache/doris/hudi/HudiScanParam.java | 194 ------
.../java/org/apache/doris/hudi/HudiScanUtils.java | 62 --
.../src/main/java/org/apache/doris/hudi/Utils.java | 21 +
.../org/apache/doris/hudi/BaseSplitReader.scala | 702 +++++++++++++++++++++
.../apache/doris/hudi/HoodieRecordIterator.scala | 142 +++++
.../apache/doris/hudi/MORSnapshotSplitReader.scala | 183 ++++++
fe/be-java-extensions/java-common/pom.xml | 4 +
.../org/apache/doris/common/jni/JniScanner.java | 20 +-
.../apache/doris/common/jni/MockJniScanner.java | 7 +-
.../apache/doris/common/jni/vec/ColumnValue.java | 2 +
.../doris/common/jni/vec/NativeColumnValue.java | 48 ++
.../apache/doris/common/jni/vec/ScanPredicate.java | 5 +
.../apache/doris/common/jni/vec/TableSchema.java | 83 +++
.../apache/doris/common/jni/vec/VectorColumn.java | 35 +-
.../apache/doris/common/jni/vec/VectorTable.java | 5 +
.../doris/maxcompute/MaxComputeColumnValue.java | 10 +-
.../org/apache/doris/paimon/PaimonColumnValue.java | 7 +-
fe/be-java-extensions/pom.xml | 1 +
fe/fe-core/pom.xml | 8 +-
.../org/apache/doris/analysis/DataDescription.java | 10 +
.../apache/doris/analysis/ImportColumnDesc.java | 4 +
.../apache/doris/analysis/NativeInsertStmt.java | 36 +-
.../org/apache/doris/analysis/S3TvfLoadStmt.java | 423 +++++++++++++
.../java/org/apache/doris/analysis/SelectStmt.java | 5 +
.../org/apache/doris/analysis/UnifiedLoadStmt.java | 13 +-
.../java/org/apache/doris/catalog/HudiUtils.java | 3 +
.../property/constants/S3Properties.java | 2 +
.../java/org/apache/doris/qe/StmtExecutor.java | 7 +-
.../ExternalFileTableValuedFunction.java | 9 +-
.../doris/tablefunction/S3TableValuedFunction.java | 41 +-
.../apache/doris/analysis/S3TvfLoadStmtTest.java | 245 +++++++
.../datasource/property/PropertyConverterTest.java | 4 +-
gensrc/thrift/PlanNodes.thrift | 1 +
.../load_p2/broker_load/test_broker_load.groovy | 63 --
.../broker_load/test_tvf_based_broker_load.groovy | 328 ++++++++++
58 files changed, 3519 insertions(+), 736 deletions(-)
create mode 100644 be/src/vec/exec/scan/avro_jni_reader.cpp
copy be/src/vec/exec/scan/{hudi_jni_reader.h => avro_jni_reader.h} (65%)
copy fe/be-java-extensions/{hudi-scanner => avro-scanner}/pom.xml (51%)
copy
fe/be-java-extensions/{hudi-scanner/src/main/java/org/apache/doris/hudi/HudiColumnValue.java
=> avro-scanner/src/main/java/org/apache/doris/avro/AvroColumnValue.java} (53%)
create mode 100644
fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroJNIScanner.java
create mode 100644
fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroProperties.java
create mode 100644
fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroReader.java
create mode 100644
fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/HDFSFileReader.java
create mode 100644
fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3FileReader.java
copy fe/be-java-extensions/{hudi-scanner =>
avro-scanner}/src/main/resources/package.xml (100%)
delete mode 100644
fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiScanParam.java
delete mode 100644
fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiScanUtils.java
create mode 100644
fe/be-java-extensions/hudi-scanner/src/main/scala/org/apache/doris/hudi/BaseSplitReader.scala
create mode 100644
fe/be-java-extensions/hudi-scanner/src/main/scala/org/apache/doris/hudi/HoodieRecordIterator.scala
create mode 100644
fe/be-java-extensions/hudi-scanner/src/main/scala/org/apache/doris/hudi/MORSnapshotSplitReader.scala
create mode 100644
fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/NativeColumnValue.java
create mode 100644
fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/TableSchema.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/analysis/S3TvfLoadStmt.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/analysis/S3TvfLoadStmtTest.java
create mode 100644
regression-test/suites/load_p2/broker_load/test_tvf_based_broker_load.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
