This is an automated email from the ASF dual-hosted git repository.
yihua pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
from 924c30c7ea [HUDI-4469] Flip reuse flag to true in
HoodieBackedTableMetadata to improve file listing (#6214)
add cdaec5a8da [HUDI-4186] Support Hudi with Spark 3.3.0 (#5943)
No new revisions were added by this update.
Summary of changes:
.github/workflows/bot.yml | 5 +-
.../apache/hudi/config/HoodieStorageConfig.java | 15 ++
.../org/apache/hudi/config/HoodieWriteConfig.java | 4 +
.../HoodieSparkBootstrapSchemaProvider.java | 18 +-
.../storage/row/HoodieRowParquetWriteSupport.java | 1 +
.../scala/org/apache/hudi/HoodieSparkUtils.scala | 2 +
.../org/apache/hudi/SparkAdapterSupport.scala | 4 +-
.../org/apache/spark/sql/hudi/SparkAdapter.scala | 48 ++++-
.../quickstart/TestHoodieFlinkQuickstart.java | 2 +
hudi-examples/hudi-examples-spark/pom.xml | 6 +
hudi-spark-datasource/README.md | 17 +-
.../org/apache/hudi/BaseFileOnlyRelation.scala | 6 +-
.../spark/sql/hudi/analysis/HoodieAnalysis.scala | 36 ++--
.../hudi/command/DeleteHoodieTableCommand.scala | 7 +-
.../spark/sql/parser/HoodieCommonSqlParser.scala | 8 +
.../spark/sql/hudi/HoodieSparkSqlTestBase.scala | 5 +-
.../sql/hudi/TestAlterTableDropPartition.scala | 11 +-
.../hudi/TestNestedSchemaPruningOptimization.scala | 1 +
.../sql/hudi/procedure/TestCallCommandParser.scala | 11 +-
.../org/apache/hudi/Spark2HoodieFileScanRDD.scala} | 13 +-
.../apache/spark/sql/adapter/Spark2Adapter.scala | 35 +++-
.../apache/hudi/spark3/internal/ReflectUtil.java | 2 +-
.../spark/sql/adapter/BaseSpark3Adapter.scala | 17 --
.../apache/hudi/Spark31HoodieFileScanRDD.scala} | 13 +-
.../apache/spark/sql/adapter/Spark3_1Adapter.scala | 37 +++-
.../datasources/Spark31NestedSchemaPruning.scala} | 2 +-
... => Spark312ResolveHudiAlterTableCommand.scala} | 2 +-
...scala => HoodieSpark3_1ExtendedSqlParser.scala} | 2 +-
.../{hudi-spark3 => hudi-spark3.2.x}/pom.xml | 65 +++---
.../src/main/antlr4/imports/SqlBase.g4 | 0
.../apache/hudi/spark/sql/parser/HoodieSqlBase.g4 | 0
...Spark32HoodieVectorizedParquetRecordReader.java | 0
...org.apache.spark.sql.sources.DataSourceRegister | 0
.../apache/hudi/Spark32HoodieFileScanRDD.scala} | 13 +-
.../org/apache/hudi/Spark3DefaultSource.scala | 0
.../sql/HoodieSpark32CatalystExpressionUtils.scala | 0
.../spark/sql/HoodieSpark32CatalystPlanUtils.scala | 0
.../apache/spark/sql/adapter/Spark3_2Adapter.scala | 34 ++-
.../apache/spark/sql/avro/AvroDeserializer.scala | 0
.../org/apache/spark/sql/avro/AvroSerializer.scala | 0
.../org/apache/spark/sql/avro/AvroUtils.scala | 0
.../sql/avro/HoodieSpark3_2AvroDeserializer.scala | 0
.../sql/avro/HoodieSpark3_2AvroSerializer.scala | 0
.../plans/logical/TimeTravelRelation.scala | 0
.../sql/connector/catalog/HoodieIdentifier.scala | 0
.../datasources/Spark32NestedSchemaPruning.scala} | 2 +-
.../parquet/Spark32DataSourceUtils.scala | 0
.../parquet/Spark32HoodieParquetFileFormat.scala | 0
.../Spark32ResolveHudiAlterTableCommand.scala} | 2 +-
.../sql/hudi/analysis/HoodieSpark3Analysis.scala | 0
.../spark/sql/hudi/catalog/BasicStagedTable.scala | 0
.../spark/sql/hudi/catalog/HoodieCatalog.scala | 4 +-
.../sql/hudi/catalog/HoodieInternalV2Table.scala | 0
.../spark/sql/hudi/catalog/HoodieStagedTable.scala | 0
.../spark/sql/hudi/catalog/TableCreationMode.java | 0
.../spark/sql/hudi/command/AlterTableCommand.scala | 0
.../HoodieSpark3_2ExtendedSqlAstBuilder.scala | 0
.../parser/HoodieSpark3_2ExtendedSqlParser.scala | 0
.../TestHoodieBulkInsertDataInternalWriter.java | 0
.../TestHoodieDataSourceInternalBatchWrite.java | 0
.../hudi/spark3/internal/TestReflectUtil.java | 0
.../test/resources/log4j-surefire-quiet.properties | 0
.../src/test/resources/log4j-surefire.properties | 0
.../{hudi-spark3 => hudi-spark3.3.x}/pom.xml | 14 +-
.../src/main/antlr4/imports/SqlBase.g4 | 0
.../apache/hudi/spark/sql/parser/HoodieSqlBase.g4 | 0
...park33HoodieVectorizedParquetRecordReader.java} | 4 +-
...org.apache.spark.sql.sources.DataSourceRegister | 0
.../apache/hudi/Spark33HoodieFileScanRDD.scala} | 13 +-
.../org/apache/hudi/Spark3DefaultSource.scala | 0
.../HoodieSpark33CatalystExpressionUtils.scala} | 6 +-
.../sql/HoodieSpark33CatalystPlanUtils.scala} | 2 +-
.../apache/spark/sql/adapter/Spark3_3Adapter.scala | 80 ++++++++
.../apache/spark/sql/avro/AvroDeserializer.scala | 171 +++++++---------
.../org/apache/spark/sql/avro/AvroSerializer.scala | 151 +++++---------
.../org/apache/spark/sql/avro/AvroUtils.scala | 228 +++++++++++++++++++++
.../sql/avro/HoodieSpark3_3AvroDeserializer.scala} | 2 +-
.../sql/avro/HoodieSpark3_3AvroSerializer.scala} | 2 +-
.../plans/logical/TimeTravelRelation.scala | 0
.../sql/connector/catalog/HoodieIdentifier.scala | 0
.../datasources/Spark33NestedSchemaPruning.scala} | 12 +-
.../parquet/Spark33DataSourceUtils.scala} | 2 +-
.../parquet/Spark33HoodieParquetFileFormat.scala} | 18 +-
.../Spark33ResolveHudiAlterTableCommand.scala} | 4 +-
.../sql/hudi/analysis/HoodieSpark3Analysis.scala | 16 +-
.../spark/sql/hudi/catalog/BasicStagedTable.scala | 0
.../spark/sql/hudi/catalog/HoodieCatalog.scala | 2 +-
.../sql/hudi/catalog/HoodieInternalV2Table.scala | 0
.../spark/sql/hudi/catalog/HoodieStagedTable.scala | 0
.../spark/sql/hudi/catalog/TableCreationMode.java | 0
.../spark/sql/hudi/command/AlterTableCommand.scala | 0
.../HoodieSpark3_3ExtendedSqlAstBuilder.scala} | 27 ++-
.../parser/HoodieSpark3_3ExtendedSqlParser.scala} | 8 +-
.../TestHoodieBulkInsertDataInternalWriter.java | 0
.../TestHoodieDataSourceInternalBatchWrite.java | 0
.../hudi/spark3/internal/TestReflectUtil.java | 0
.../test/resources/log4j-surefire-quiet.properties | 0
.../src/test/resources/log4j-surefire.properties | 0
pom.xml | 58 +++++-
99 files changed, 881 insertions(+), 389 deletions(-)
copy
hudi-spark-datasource/{hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileScanRDD.scala
=> hudi-spark2/src/main/scala/org/apache/hudi/Spark2HoodieFileScanRDD.scala}
(71%)
copy
hudi-spark-datasource/{hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileScanRDD.scala
=>
hudi-spark3.1.x/src/main/scala/org/apache/hudi/Spark31HoodieFileScanRDD.scala}
(71%)
copy
hudi-spark-datasource/{hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/NestedSchemaPruning.scala
=>
hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark31NestedSchemaPruning.scala}
(99%)
rename
hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/hudi/{ResolveHudiAlterTableCommand312.scala
=> Spark312ResolveHudiAlterTableCommand.scala} (99%)
rename
hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/parser/{HoodieSpark312ExtendedSqlParser.scala
=> HoodieSpark3_1ExtendedSqlParser.scala} (95%)
copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.2.x}/pom.xml (92%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/antlr4/imports/SqlBase.g4 (100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/antlr4/org/apache/hudi/spark/sql/parser/HoodieSqlBase.g4
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark32HoodieVectorizedParquetRecordReader.java
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
(100%)
copy
hudi-spark-datasource/{hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileScanRDD.scala
=>
hudi-spark3.2.x/src/main/scala/org/apache/hudi/Spark32HoodieFileScanRDD.scala}
(71%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/hudi/Spark3DefaultSource.scala (100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/HoodieSpark32CatalystExpressionUtils.scala
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/HoodieSpark32CatalystPlanUtils.scala
(100%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/adapter/Spark3_2Adapter.scala
(61%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
(100%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala (100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_2AvroDeserializer.scala
(100%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_2AvroSerializer.scala
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TimeTravelRelation.scala
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/connector/catalog/HoodieIdentifier.scala
(100%)
copy
hudi-spark-datasource/{hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/NestedSchemaPruning.scala
=>
hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark32NestedSchemaPruning.scala}
(99%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32DataSourceUtils.scala
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32HoodieParquetFileFormat.scala
(100%)
copy
hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/hudi/ResolveHudiAlterTableCommandSpark32.scala
=>
hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/hudi/Spark32ResolveHudiAlterTableCommand.scala}
(98%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark3Analysis.scala
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/BasicStagedTable.scala
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieCatalog.scala
(99%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieInternalV2Table.scala
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieStagedTable.scala
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/TableCreationMode.java
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/hudi/command/AlterTableCommand.scala
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_2ExtendedSqlAstBuilder.scala
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_2ExtendedSqlParser.scala
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieBulkInsertDataInternalWriter.java
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieDataSourceInternalBatchWrite.java
(100%)
copy hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java
(100%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/test/resources/log4j-surefire-quiet.properties (100%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.2.x}/src/test/resources/log4j-surefire.properties (100%)
rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3.3.x}/pom.xml (96%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/antlr4/imports/SqlBase.g4 (100%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/antlr4/org/apache/hudi/spark/sql/parser/HoodieSqlBase.g4
(100%)
rename
hudi-spark-datasource/{hudi-spark3/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark32HoodieVectorizedParquetRecordReader.java
=>
hudi-spark3.3.x/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark33HoodieVectorizedParquetRecordReader.java}
(98%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
(100%)
rename
hudi-spark-datasource/{hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileScanRDD.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/hudi/Spark33HoodieFileScanRDD.scala}
(67%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/scala/org/apache/hudi/Spark3DefaultSource.scala (100%)
rename
hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/HoodieSpark32CatalystExpressionUtils.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystExpressionUtils.scala}
(94%)
rename
hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/HoodieSpark32CatalystPlanUtils.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystPlanUtils.scala}
(95%)
create mode 100644
hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_3Adapter.scala
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
(82%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
(73%)
create mode 100644
hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala
rename
hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_2AvroDeserializer.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_3AvroDeserializer.scala}
(95%)
copy
hudi-spark-datasource/{hudi-spark2/src/main/scala/org/apache/spark/sql/avro/HoodieSpark2_4AvroSerializer.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_3AvroSerializer.scala}
(94%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TimeTravelRelation.scala
(100%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/connector/catalog/HoodieIdentifier.scala
(100%)
rename
hudi-spark-datasource/{hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/NestedSchemaPruning.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark33NestedSchemaPruning.scala}
(95%)
rename
hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32DataSourceUtils.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33DataSourceUtils.scala}
(99%)
rename
hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32HoodieParquetFileFormat.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33HoodieParquetFileFormat.scala}
(97%)
rename
hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/hudi/ResolveHudiAlterTableCommandSpark32.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/Spark33ResolveHudiAlterTableCommand.scala}
(95%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark3Analysis.scala
(94%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/BasicStagedTable.scala
(100%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieCatalog.scala
(99%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieInternalV2Table.scala
(100%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieStagedTable.scala
(100%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/hudi/catalog/TableCreationMode.java
(100%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/main/scala/org/apache/spark/sql/hudi/command/AlterTableCommand.scala
(100%)
rename
hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_2ExtendedSqlAstBuilder.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlAstBuilder.scala}
(99%)
rename
hudi-spark-datasource/{hudi-spark3/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_2ExtendedSqlParser.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlParser.scala}
(95%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieBulkInsertDataInternalWriter.java
(100%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieDataSourceInternalBatchWrite.java
(100%)
rename hudi-spark-datasource/{hudi-spark3 =>
hudi-spark3.3.x}/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java
(100%)
copy {hudi-common =>
hudi-spark-datasource/hudi-spark3.3.x}/src/test/resources/log4j-surefire-quiet.properties
(100%)
copy {hudi-client/hudi-client-common =>
hudi-spark-datasource/hudi-spark3.3.x}/src/test/resources/log4j-surefire.properties
(100%)
