This is an automated email from the ASF dual-hosted git repository.
yihua pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
from c849866d6c7 [HUDI-3545] [UBER] Make HoodieAvroWriteSupport class
configurable (#8590)
add 96ca7aac693 [HUDI-6198] Support Hudi on Spark 3.4.0 (#8885)
No new revisions were added by this update.
Summary of changes:
.github/workflows/bot.yml | 9 ++-
.../io/storage/HoodieSparkFileReaderFactory.java | 3 +
.../scala/org/apache/hudi/HoodieSparkUtils.scala | 2 +
.../org/apache/hudi/SparkAdapterSupport.scala | 4 +-
.../scala/org/apache/hudi/SparkJdbcUtils.scala | 64 ++++++++++++++++
.../spark/sql/HoodieCatalystPlansUtils.scala | 25 ++++++
.../org/apache/spark/sql/HoodieSchemaUtils.scala | 24 +++---
.../HoodieSparkPartitionedFileUtils.scala | 77 +++++++++++++++++++
.../org/apache/spark/sql/hudi/SparkAdapter.scala | 10 +++
.../org/apache/hudi/BaseFileOnlyRelation.scala | 10 ++-
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 7 +-
.../org/apache/hudi/HoodieBootstrapRelation.scala | 26 ++++---
.../org/apache/hudi/HoodieDataSourceHelper.scala | 9 +--
.../src/main/scala/org/apache/hudi/Iterators.scala | 19 ++---
.../apache/hudi/MergeOnReadSnapshotRelation.scala | 29 +++----
.../scala/org/apache/hudi/cdc/HoodieCDCRDD.scala | 16 ++--
.../parquet/HoodieParquetFileFormat.scala | 13 +++-
.../AlterHoodieTableAddColumnsCommand.scala | 37 +++++----
.../spark/sql/hudi/analysis/HoodieAnalysis.scala | 29 ++++---
.../model/TestHoodieRecordSerialization.scala | 17 +++--
.../apache/hudi/functional/TestCOWDataSource.scala | 4 +-
.../hudi/functional/TestStructuredStreaming.scala | 2 +-
.../apache/spark/sql/hudi/TestInsertTable.scala | 24 +++++-
.../apache/spark/sql/hudi/TestMergeIntoTable.scala | 2 +-
.../sql/hudi/command/index/TestIndexSyntax.scala | 17 ++++-
.../procedure/TestCopyToTempViewProcedure.scala | 15 +++-
.../spark/sql/HoodieSpark2CatalystPlanUtils.scala | 12 ++-
.../apache/spark/sql/HoodieSpark2SchemaUtils.scala | 24 ++----
.../apache/spark/sql/adapter/Spark2Adapter.scala | 4 +
.../HoodieSpark2PartitionedFileUtils.scala | 26 ++++---
.../spark/sql/HoodieSpark30CatalystPlanUtils.scala | 12 ++-
.../spark/sql/HoodieSpark30SchemaUtils.scala | 24 ++----
.../apache/spark/sql/adapter/Spark3_0Adapter.scala | 20 +++--
.../HoodieSpark30PartitionedFileUtils.scala | 43 +++++++++++
.../spark/sql/HoodieSpark31CatalystPlanUtils.scala | 12 ++-
.../spark/sql/HoodieSpark31SchemaUtils.scala | 24 ++----
.../apache/spark/sql/adapter/Spark3_1Adapter.scala | 22 +++---
.../HoodieSpark31PartitionedFileUtils.scala | 43 +++++++++++
.../spark/sql/HoodieSpark32CatalystPlanUtils.scala | 16 ++--
.../spark/sql/HoodieSpark32SchemaUtils.scala | 24 ++----
.../apache/spark/sql/adapter/Spark3_2Adapter.scala | 14 ++--
.../HoodieSpark32PartitionedFileUtils.scala | 43 +++++++++++
.../parquet/Spark32HoodieParquetFileFormat.scala} | 10 +--
...k32PlusHoodieVectorizedParquetRecordReader.java | 7 +-
.../spark/sql/HoodieSpark33CatalystPlanUtils.scala | 15 +++-
.../spark/sql/HoodieSpark33SchemaUtils.scala | 24 ++----
.../apache/spark/sql/adapter/Spark3_3Adapter.scala | 14 ++--
.../HoodieSpark33PartitionedFileUtils.scala | 43 +++++++++++
.../parquet/Spark33HoodieParquetFileFormat.scala} | 43 ++++++-----
.../{hudi-spark3.3.x => hudi-spark3.4.x}/pom.xml | 39 ++++++++--
.../src/main/antlr4/imports/SqlBase.g4 | 0
.../apache/hudi/spark/sql/parser/HoodieSqlBase.g4 | 0
...org.apache.spark.sql.sources.DataSourceRegister | 0
.../apache/hudi/Spark34HoodieFileScanRDD.scala} | 2 +-
.../spark/sql/HoodieSpark34CatalogUtils.scala} | 2 +-
.../HoodieSpark34CatalystExpressionUtils.scala} | 22 +++---
.../sql/HoodieSpark34CatalystPlanUtils.scala} | 17 +++--
.../spark/sql/HoodieSpark34SchemaUtils.scala | 24 ++----
.../spark/sql/adapter/Spark3_4Adapter.scala} | 38 ++++-----
.../apache/spark/sql/avro/AvroDeserializer.scala | 0
.../org/apache/spark/sql/avro/AvroSerializer.scala | 10 +--
.../org/apache/spark/sql/avro/AvroUtils.scala | 0
.../sql/avro/HoodieSpark3_4AvroDeserializer.scala} | 2 +-
.../sql/avro/HoodieSpark3_4AvroSerializer.scala} | 2 +-
.../HoodieSpark34PartitionedFileUtils.scala | 44 +++++++++++
.../datasources/Spark34NestedSchemaPruning.scala} | 2 +-
.../parquet/Spark34HoodieParquetFileFormat.scala} | 51 ++++++++-----
.../Spark34ResolveHudiAlterTableCommand.scala} | 4 +-
.../HoodieSpark3_4ExtendedSqlAstBuilder.scala} | 30 ++++----
.../parser/HoodieSpark3_4ExtendedSqlParser.scala} | 22 +++---
.../TestHoodieBulkInsertDataInternalWriter.java | 0
.../TestHoodieDataSourceInternalBatchWrite.java | 0
.../hudi/spark3/internal/TestReflectUtil.java | 0
.../org/apache/hudi/utilities/UtilHelpers.java | 8 +-
packaging/bundle-validation/ci_run.sh | 10 +++
pom.xml | 89 +++++++++++++++++++---
76 files changed, 1023 insertions(+), 408 deletions(-)
create mode 100644
hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkJdbcUtils.scala
copy
hudi-common/src/main/java/org/apache/hudi/common/data/HoodieAccumulator.java =>
hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/HoodieSchemaUtils.scala
(55%)
create mode 100644
hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSparkPartitionedFileUtils.scala
copy hudi-common/src/main/java/org/apache/hudi/common/util/MapUtils.java =>
hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/spark/sql/HoodieSpark2SchemaUtils.scala
(63%)
copy hudi-common/src/main/java/org/apache/hudi/common/util/MapUtils.java =>
hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark2PartitionedFileUtils.scala
(50%)
copy hudi-common/src/main/java/org/apache/hudi/common/util/MapUtils.java =>
hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark30SchemaUtils.scala
(63%)
create mode 100644
hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark30PartitionedFileUtils.scala
copy hudi-common/src/main/java/org/apache/hudi/common/util/MapUtils.java =>
hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark31SchemaUtils.scala
(63%)
create mode 100644
hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark31PartitionedFileUtils.scala
copy hudi-common/src/main/java/org/apache/hudi/common/util/MapUtils.java =>
hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/HoodieSpark32SchemaUtils.scala
(63%)
create mode 100644
hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark32PartitionedFileUtils.scala
copy
hudi-spark-datasource/{hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32PlusHoodieParquetFileFormat.scala
=>
hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32HoodieParquetFileFormat.scala}
(98%)
copy hudi-common/src/main/java/org/apache/hudi/common/util/MapUtils.java =>
hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33SchemaUtils.scala
(63%)
create mode 100644
hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark33PartitionedFileUtils.scala
copy
hudi-spark-datasource/{hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32PlusHoodieParquetFileFormat.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33HoodieParquetFileFormat.scala}
(94%)
copy hudi-spark-datasource/{hudi-spark3.3.x => hudi-spark3.4.x}/pom.xml (90%)
copy hudi-spark-datasource/{hudi-spark3.2.x =>
hudi-spark3.4.x}/src/main/antlr4/imports/SqlBase.g4 (100%)
copy hudi-spark-datasource/{hudi-spark3.2.x =>
hudi-spark3.4.x}/src/main/antlr4/org/apache/hudi/spark/sql/parser/HoodieSqlBase.g4
(100%)
copy hudi-spark-datasource/{hudi-spark3.2.x =>
hudi-spark3.4.x}/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
(100%)
copy
hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/hudi/Spark33HoodieFileScanRDD.scala
=>
hudi-spark3.4.x/src/main/scala/org/apache/hudi/Spark34HoodieFileScanRDD.scala}
(96%)
copy
hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalogUtils.scala
=>
hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34CatalogUtils.scala}
(94%)
copy
hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystExpressionUtils.scala
=>
hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34CatalystExpressionUtils.scala}
(87%)
copy
hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystPlanUtils.scala
=>
hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34CatalystPlanUtils.scala}
(84%)
copy hudi-common/src/main/java/org/apache/hudi/common/util/MapUtils.java =>
hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34SchemaUtils.scala
(64%)
copy
hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_3Adapter.scala
=>
hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_4Adapter.scala}
(80%)
copy hudi-spark-datasource/{hudi-spark3.3.x =>
hudi-spark3.4.x}/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
(100%)
copy hudi-spark-datasource/{hudi-spark3.3.x =>
hudi-spark3.4.x}/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
(99%)
copy hudi-spark-datasource/{hudi-spark3.3.x =>
hudi-spark3.4.x}/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala (100%)
copy
hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_3AvroDeserializer.scala
=>
hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_4AvroDeserializer.scala}
(95%)
copy
hudi-spark-datasource/{hudi-spark2/src/main/scala/org/apache/spark/sql/avro/HoodieSpark2_4AvroSerializer.scala
=>
hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_4AvroSerializer.scala}
(94%)
create mode 100644
hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark34PartitionedFileUtils.scala
copy
hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark33NestedSchemaPruning.scala
=>
hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark34NestedSchemaPruning.scala}
(99%)
rename
hudi-spark-datasource/{hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32PlusHoodieParquetFileFormat.scala
=>
hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark34HoodieParquetFileFormat.scala}
(92%)
copy
hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/Spark33ResolveHudiAlterTableCommand.scala
=>
hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/hudi/Spark34ResolveHudiAlterTableCommand.scala}
(97%)
copy
hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlAstBuilder.scala
=>
hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_4ExtendedSqlAstBuilder.scala}
(99%)
copy
hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlParser.scala
=>
hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_4ExtendedSqlParser.scala}
(92%)
copy hudi-spark-datasource/{hudi-spark3.3.x =>
hudi-spark3.4.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieBulkInsertDataInternalWriter.java
(100%)
copy hudi-spark-datasource/{hudi-spark3.3.x =>
hudi-spark3.4.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieDataSourceInternalBatchWrite.java
(100%)
copy hudi-spark-datasource/{hudi-spark3.3.x =>
hudi-spark3.4.x}/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java
(100%)
