This is an automated email from the ASF dual-hosted git repository.
yihua pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
from 35af64db466 [Minor] Throw exceptions when cleaner/compactor fail
(#10108)
add 874b5dec5e9 [HUDI-6806] Support Spark 3.5.0 (#9717)
No new revisions were added by this update.
Summary of changes:
.github/workflows/bot.yml | 13 +++
.../scala/org/apache/hudi/HoodieSparkUtils.scala | 2 +
.../org/apache/hudi/SparkAdapterSupport.scala | 4 +-
.../scala/org/apache/spark/sql/DataFrameUtil.scala | 6 +-
.../spark/sql/HoodieCatalystExpressionUtils.scala | 16 ++--
.../org/apache/spark/sql/HoodieSchemaUtils.scala | 9 +++
.../org/apache/spark/sql/HoodieUnsafeUtils.scala | 13 +--
.../HoodieSparkPartitionedFileUtils.scala | 20 +++--
.../org/apache/spark/sql/hudi/SparkAdapter.scala | 5 +-
.../org/apache/hudi/avro/TestHoodieAvroUtils.java | 4 +-
.../hudi/common/util/TestClusteringUtils.java | 2 +
.../dag/nodes/BaseValidateDatasetNode.java | 13 +--
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 4 +-
.../scala/org/apache/hudi/HoodieCDCFileIndex.scala | 2 +-
.../scala/org/apache/hudi/HoodieFileIndex.scala | 9 ++-
.../apache/hudi/HoodieIncrementalFileIndex.scala | 9 ++-
.../datasources/HoodieInMemoryFileIndex.scala | 5 +-
.../hudi/testutils/SparkDatasetTestUtils.java | 19 ++---
hudi-spark-datasource/hudi-spark/pom.xml | 30 +++++++
.../spark/sql/hudi/analysis/HoodieAnalysis.scala | 19 ++++-
.../hudi/command/CallProcedureHoodieCommand.scala | 6 +-
.../hudi/command/CompactionHoodiePathCommand.scala | 5 +-
.../command/CompactionHoodieTableCommand.scala | 5 +-
.../command/CompactionShowHoodiePathCommand.scala | 5 +-
.../command/CompactionShowHoodieTableCommand.scala | 5 +-
.../command/InsertIntoHoodieTableCommand.scala | 10 ++-
.../TestBulkInsertInternalPartitionerForRows.java | 0
.../TestHoodieDatasetBulkInsertHelper.java | 19 ++---
.../row/TestHoodieInternalRowParquetWriter.java | 0
.../io/storage/row/TestHoodieRowCreateHandle.java | 14 +++-
.../hudi/testutils/KeyGeneratorTestUtilities.java | 20 ++---
.../org/apache/hudi/TestAvroConversionUtils.scala | 2 +-
.../read/TestHoodieFileGroupReaderOnSpark.scala | 9 ++-
.../apache/spark/sql/hudi/TestInsertTable.scala | 22 +++++-
hudi-spark-datasource/hudi-spark2/pom.xml | 8 ++
.../sql/HoodieSpark2CatalystExpressionUtils.scala | 7 +-
.../apache/spark/sql/HoodieSpark2SchemaUtils.scala | 6 ++
.../apache/spark/sql/adapter/Spark2Adapter.scala | 7 +-
.../HoodieSpark2PartitionedFileUtils.scala | 12 ++-
.../HoodieBulkInsertInternalWriterTestBase.java | 0
.../apache/hudi/spark3/internal/ReflectUtil.java | 8 +-
.../spark/sql/adapter/BaseSpark3Adapter.scala | 6 +-
hudi-spark-datasource/hudi-spark3.0.x/pom.xml | 15 ++++
.../sql/HoodieSpark30CatalystExpressionUtils.scala | 7 +-
.../spark/sql/HoodieSpark30SchemaUtils.scala | 6 ++
.../HoodieSpark30PartitionedFileUtils.scala | 12 ++-
.../HoodieBulkInsertInternalWriterTestBase.java | 0
.../TestHoodieBulkInsertDataInternalWriter.java | 0
.../TestHoodieDataSourceInternalBatchWrite.java | 0
hudi-spark-datasource/hudi-spark3.1.x/pom.xml | 15 ++++
.../sql/HoodieSpark31CatalystExpressionUtils.scala | 8 +-
.../spark/sql/HoodieSpark31SchemaUtils.scala | 6 ++
.../HoodieSpark31PartitionedFileUtils.scala | 12 ++-
.../HoodieBulkInsertInternalWriterTestBase.java | 0
.../TestHoodieBulkInsertDataInternalWriter.java | 0
.../TestHoodieDataSourceInternalBatchWrite.java | 0
hudi-spark-datasource/hudi-spark3.2.x/pom.xml | 8 +-
.../sql/HoodieSpark32CatalystExpressionUtils.scala | 7 +-
.../spark/sql/HoodieSpark32SchemaUtils.scala | 6 ++
.../HoodieSpark32PartitionedFileUtils.scala | 12 ++-
.../parquet/Spark32DataSourceUtils.scala} | 2 +-
.../Spark32LegacyHoodieParquetFileFormat.scala | 10 +--
.../sql/hudi/analysis/HoodieSpark32Analysis.scala | 66 ++++++++++++++++
.../HoodieBulkInsertInternalWriterTestBase.java | 0
.../TestHoodieBulkInsertDataInternalWriter.java | 0
.../TestHoodieDataSourceInternalBatchWrite.java | 0
.../hudi/analysis/HoodieSpark32PlusAnalysis.scala | 28 -------
.../sql/HoodieSpark33CatalystExpressionUtils.scala | 9 ++-
.../spark/sql/HoodieSpark33SchemaUtils.scala | 6 ++
.../HoodieSpark33PartitionedFileUtils.scala | 12 ++-
.../parquet/Spark33DataSourceUtils.scala} | 2 +-
.../Spark33LegacyHoodieParquetFileFormat.scala | 10 +--
.../sql/hudi/analysis/HoodieSpark33Analysis.scala | 66 ++++++++++++++++
.../HoodieBulkInsertInternalWriterTestBase.java | 0
.../hudi/spark3/internal/TestReflectUtil.java | 3 +-
.../sql/HoodieSpark34CatalystExpressionUtils.scala | 7 +-
.../spark/sql/HoodieSpark34SchemaUtils.scala | 6 ++
.../HoodieSpark34PartitionedFileUtils.scala | 12 ++-
.../parquet/Spark34DataSourceUtils.scala} | 2 +-
.../Spark34LegacyHoodieParquetFileFormat.scala | 10 +--
.../sql/hudi/analysis/HoodieSpark34Analysis.scala | 66 ++++++++++++++++
.../HoodieBulkInsertInternalWriterTestBase.java | 0
.../hudi/spark3/internal/TestReflectUtil.java | 3 +-
.../{hudi-spark3.4.x => hudi-spark3.5.x}/pom.xml | 14 ++--
.../src/main/antlr4/imports/SqlBase.g4 | 0
.../apache/hudi/spark/sql/parser/HoodieSqlBase.g4 | 0
...org.apache.spark.sql.sources.DataSourceRegister | 0
.../apache/hudi/Spark35HoodieFileScanRDD.scala} | 2 +-
.../spark/sql/HoodieSpark35CatalogUtils.scala} | 2 +-
.../HoodieSpark35CatalystExpressionUtils.scala} | 11 ++-
.../sql/HoodieSpark35CatalystPlanUtils.scala} | 2 +-
.../spark/sql/HoodieSpark35SchemaUtils.scala} | 11 ++-
.../spark/sql/adapter/Spark3_5Adapter.scala} | 37 +++++----
.../apache/spark/sql/avro/AvroDeserializer.scala | 2 +-
.../org/apache/spark/sql/avro/AvroSerializer.scala | 3 +-
.../org/apache/spark/sql/avro/AvroUtils.scala | 0
.../sql/avro/HoodieSpark3_5AvroDeserializer.scala} | 5 +-
.../sql/avro/HoodieSpark3_5AvroSerializer.scala} | 2 +-
.../HoodieSpark35PartitionedFileUtils.scala} | 14 +++-
.../datasources/Spark35NestedSchemaPruning.scala} | 6 +-
.../parquet/Spark35DataSourceUtils.scala} | 9 +--
.../Spark35LegacyHoodieParquetFileFormat.scala} | 24 +++---
.../Spark35ResolveHudiAlterTableCommand.scala} | 2 +-
.../sql/hudi/analysis/HoodieSpark35Analysis.scala | 66 ++++++++++++++++
.../HoodieSpark3_5ExtendedSqlAstBuilder.scala} | 10 ++-
.../parser/HoodieSpark3_5ExtendedSqlParser.scala} | 4 +-
.../HoodieBulkInsertInternalWriterTestBase.java | 0
.../TestHoodieBulkInsertDataInternalWriter.java | 0
.../TestHoodieDataSourceInternalBatchWrite.java | 0
.../hudi/spark3/internal/TestReflectUtil.java | 11 +--
...ark332.sh => build_flink1180hive313spark350.sh} | 6 +-
packaging/bundle-validation/ci_run.sh | 10 +++
packaging/bundle-validation/run_docker_java17.sh | 10 +++
packaging/hudi-utilities-bundle/pom.xml | 6 ++
packaging/hudi-utilities-slim-bundle/pom.xml | 6 ++
pom.xml | 92 +++++++++++++++++++---
116 files changed, 902 insertions(+), 279 deletions(-)
rename {hudi-client/hudi-spark-client =>
hudi-spark-datasource/hudi-spark-common}/src/test/java/org/apache/hudi/testutils/SparkDatasetTestUtils.java
(94%)
rename {hudi-client/hudi-spark-client =>
hudi-spark-datasource/hudi-spark}/src/test/java/org/apache/hudi/execution/bulkinsert/TestBulkInsertInternalPartitionerForRows.java
(100%)
rename {hudi-client/hudi-spark-client =>
hudi-spark-datasource/hudi-spark}/src/test/java/org/apache/hudi/io/storage/row/TestHoodieInternalRowParquetWriter.java
(100%)
rename {hudi-client/hudi-spark-client =>
hudi-spark-datasource/hudi-spark}/src/test/java/org/apache/hudi/io/storage/row/TestHoodieRowCreateHandle.java
(94%)
copy hudi-spark-datasource/{hudi-spark-common =>
hudi-spark2}/src/test/java/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java
(100%)
copy hudi-spark-datasource/{hudi-spark-common =>
hudi-spark3.0.x}/src/test/java/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java
(100%)
copy hudi-spark-datasource/{hudi-spark3-common =>
hudi-spark3.0.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieBulkInsertDataInternalWriter.java
(100%)
copy hudi-spark-datasource/{hudi-spark3-common =>
hudi-spark3.0.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieDataSourceInternalBatchWrite.java
(100%)
copy hudi-spark-datasource/{hudi-spark-common/src/test/java =>
hudi-spark3.1.x/src/test}/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java
(100%)
copy hudi-spark-datasource/{hudi-spark3-common/src/test/java =>
hudi-spark3.1.x/src/test}/org/apache/hudi/spark3/internal/TestHoodieBulkInsertDataInternalWriter.java
(100%)
copy hudi-spark-datasource/{hudi-spark3-common/src/test/java =>
hudi-spark3.1.x/src/test}/org/apache/hudi/spark3/internal/TestHoodieDataSourceInternalBatchWrite.java
(100%)
copy
hudi-spark-datasource/{hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32PlusDataSourceUtils.scala
=>
hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32DataSourceUtils.scala}
(98%)
create mode 100644
hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark32Analysis.scala
copy hudi-spark-datasource/{hudi-spark-common =>
hudi-spark3.2.x}/src/test/java/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java
(100%)
rename hudi-spark-datasource/{hudi-spark3-common =>
hudi-spark3.2.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieBulkInsertDataInternalWriter.java
(100%)
rename hudi-spark-datasource/{hudi-spark3-common =>
hudi-spark3.2.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieDataSourceInternalBatchWrite.java
(100%)
copy
hudi-spark-datasource/{hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32PlusDataSourceUtils.scala
=>
hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33DataSourceUtils.scala}
(98%)
create mode 100644
hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark33Analysis.scala
copy hudi-spark-datasource/{hudi-spark-common =>
hudi-spark3.3.x}/src/test/java/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java
(100%)
copy
hudi-spark-datasource/{hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32PlusDataSourceUtils.scala
=>
hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark34DataSourceUtils.scala}
(98%)
create mode 100644
hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark34Analysis.scala
copy hudi-spark-datasource/{hudi-spark-common =>
hudi-spark3.4.x}/src/test/java/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java
(100%)
copy hudi-spark-datasource/{hudi-spark3.4.x => hudi-spark3.5.x}/pom.xml (96%)
copy hudi-spark-datasource/{hudi-spark3.2.x =>
hudi-spark3.5.x}/src/main/antlr4/imports/SqlBase.g4 (100%)
copy hudi-spark-datasource/{hudi-spark3.2.x =>
hudi-spark3.5.x}/src/main/antlr4/org/apache/hudi/spark/sql/parser/HoodieSqlBase.g4
(100%)
copy hudi-spark-datasource/{hudi-spark3.2.x =>
hudi-spark3.5.x}/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
(100%)
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/hudi/Spark34HoodieFileScanRDD.scala
=>
hudi-spark3.5.x/src/main/scala/org/apache/hudi/Spark35HoodieFileScanRDD.scala}
(96%)
copy
hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalogUtils.scala
=>
hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/HoodieSpark35CatalogUtils.scala}
(94%)
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34CatalystExpressionUtils.scala
=>
hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/HoodieSpark35CatalystExpressionUtils.scala}
(94%)
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34CatalystPlanUtils.scala
=>
hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/HoodieSpark35CatalystPlanUtils.scala}
(98%)
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34SchemaUtils.scala
=>
hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/HoodieSpark35SchemaUtils.scala}
(76%)
copy
hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_3Adapter.scala
=>
hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_5Adapter.scala}
(83%)
copy hudi-spark-datasource/{hudi-spark3.4.x =>
hudi-spark3.5.x}/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
(99%)
copy hudi-spark-datasource/{hudi-spark3.4.x =>
hudi-spark3.5.x}/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
(99%)
copy hudi-spark-datasource/{hudi-spark3.3.x =>
hudi-spark3.5.x}/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala (100%)
copy
hudi-spark-datasource/{hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_2AvroDeserializer.scala
=>
hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_5AvroDeserializer.scala}
(87%)
copy
hudi-spark-datasource/{hudi-spark2/src/main/scala/org/apache/spark/sql/avro/HoodieSpark2_4AvroSerializer.scala
=>
hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_5AvroSerializer.scala}
(94%)
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark34PartitionedFileUtils.scala
=>
hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark35PartitionedFileUtils.scala}
(75%)
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark34NestedSchemaPruning.scala
=>
hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark35NestedSchemaPruning.scala}
(98%)
rename
hudi-spark-datasource/{hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32PlusDataSourceUtils.scala
=>
hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark35DataSourceUtils.scala}
(91%)
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark34LegacyHoodieParquetFileFormat.scala
=>
hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark35LegacyHoodieParquetFileFormat.scala}
(95%)
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/hudi/Spark34ResolveHudiAlterTableCommand.scala
=>
hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/hudi/Spark35ResolveHudiAlterTableCommand.scala}
(98%)
create mode 100644
hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark35Analysis.scala
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_4ExtendedSqlAstBuilder.scala
=>
hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_5ExtendedSqlAstBuilder.scala}
(99%)
copy
hudi-spark-datasource/{hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_4ExtendedSqlParser.scala
=>
hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_5ExtendedSqlParser.scala}
(98%)
rename hudi-spark-datasource/{hudi-spark-common =>
hudi-spark3.5.x}/src/test/java/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java
(100%)
copy hudi-spark-datasource/{hudi-spark3.3.x =>
hudi-spark3.5.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieBulkInsertDataInternalWriter.java
(100%)
copy hudi-spark-datasource/{hudi-spark3.3.x =>
hudi-spark3.5.x}/src/test/java/org/apache/hudi/spark3/internal/TestHoodieDataSourceInternalBatchWrite.java
(100%)
rename hudi-spark-datasource/{hudi-spark3-common =>
hudi-spark3.5.x}/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java
(90%)
copy packaging/bundle-validation/base/{build_flink1180hive313spark332.sh =>
build_flink1180hive313spark350.sh} (85%)
