This is an automated email from the ASF dual-hosted git repository.
yihua pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
from d10310f4d9f3 fix: fixing hivesync tool in docker (#13957)
add be6f08d56643 feat(metadata): Improve Logical Type Handling on Col
Stats (#13711)
No new revisions were added by this update.
Summary of changes:
.../apache/hudi/index/bloom/HoodieBloomIndex.java | 7 +-
.../org/apache/hudi/io/HoodieAppendHandle.java | 10 +-
.../org/apache/hudi/io/HoodieBinaryCopyHandle.java | 7 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 4 +-
.../table/upgrade/NineToEightDowngradeHandler.java | 4 +-
.../hudi/table/upgrade/UpgradeDowngradeUtils.java | 19 +-
.../upgrade/TestNineToEightDowngradeHandler.java | 2 +-
.../io/log/block/HoodieFlinkParquetDataBlock.java | 7 +-
.../apache/hudi/io/v2/RowDataLogWriteHandle.java | 14 +-
.../client/utils/SparkInternalSchemaConverter.java | 11 +-
.../client/utils/SparkMetadataWriterUtils.java | 33 +-
.../hudi/common/model/HoodieSparkRecord.java | 4 +-
.../storage/row/HoodieRowParquetWriteSupport.java | 4 +-
.../SparkHoodieBackedTableMetadataWriter.java | 2 +-
.../apache/hudi/stats/SparkValueMetadataUtils.java | 157 +++++
.../bootstrap/ParquetBootstrapMetadataHandler.java | 4 +-
.../hudi/BaseSparkInternalRecordContext.java | 33 +-
.../org/apache/hudi/client/TestWriteStatus.java | 5 +-
.../functional/TestExternalPathHandling.java | 7 +-
.../functional/TestHoodieBackedTableMetadata.java | 43 +-
hudi-common/src/main/avro/HoodieMetadata.avsc | 32 +-
.../main/java/org/apache/hudi/ParquetAdapter.java | 31 +-
.../java/org/apache/hudi/avro/AvroSchemaUtils.java | 33 +-
.../apache/hudi/avro/ConvertingGenericData.java | 24 +-
.../java/org/apache/hudi/avro/HoodieAvroUtils.java | 33 +-
.../apache/hudi/avro/HoodieAvroWrapperUtils.java | 154 ++++-
.../org/apache/hudi/avro/processors/Parser.java | 10 +-
.../apache/hudi/common/model/HoodieWriteStat.java | 1 +
.../org/apache/hudi/common/util/DateTimeUtils.java | 44 ++
.../apache/hudi/common/util/FileFormatUtils.java | 30 +-
.../org/apache/hudi/common/util/HFileUtils.java | 5 +-
.../org/apache/hudi/expression/Comparators.java | 3 +
.../java/org/apache/hudi/internal/schema/Type.java | 11 +-
.../org/apache/hudi/internal/schema/Types.java | 211 ++++++-
.../convert/AvroInternalSchemaConverter.java | 69 ++-
.../hudi/internal/schema/utils/Conversions.java | 2 +
.../internal/schema/utils/SchemaChangeUtils.java | 79 ++-
.../hudi/internal/schema/utils/SerDeHelper.java | 25 +
.../io/storage/ColumnRangeMetadataProvider.java | 5 +-
.../apache/hudi/metadata/HoodieIndexVersion.java | 49 +-
.../hudi/metadata/HoodieMetadataPayload.java | 16 +-
.../hudi/metadata/HoodieTableMetadataUtil.java | 325 ++++++----
.../hudi/metadata/MetadataPartitionType.java | 9 +-
.../model => stats}/HoodieColumnRangeMetadata.java | 73 ++-
.../java/org/apache/hudi/stats/ValueMetadata.java | 266 +++++++++
.../main/java/org/apache/hudi/stats/ValueType.java | 665 +++++++++++++++++++++
.../parquet/schema/LogicalTypeParquetAdapter.java | 92 +++
.../parquet/schema/OriginalTypeParquetAdapter.java | 83 +++
.../hudi/common/model/TestHoodieWriteStat.java | 4 +-
.../common/testutils/HoodieTestDataGenerator.java | 206 ++++++-
.../apache/hudi/common/util/TestBaseFileUtils.java | 27 +-
.../hudi/metadata/TestHoodieIndexVersion.java | 6 +-
.../org/apache/hudi/stats/TestValueMetadata.java | 60 ++
.../java/org/apache/hudi/stats/TestValueType.java | 57 ++
.../apache/hudi/utils/TestAvroSchemaConverter.java | 4 +-
.../common/table/ParquetTableSchemaResolver.java | 12 +-
.../java/org/apache/hudi/common/util/OrcUtils.java | 5 +-
.../org/apache/hudi/common/util/ParquetUtils.java | 76 ++-
.../io/hadoop/HoodieAvroFileWriterFactory.java | 5 +-
.../avro/AvroSchemaConverterWithTimestampNTZ.java | 598 ++++++++++++++++++
.../avro/HoodieAvroParquetSchemaConverter.java | 57 ++
.../parquet/avro/NativeAvroSchemaConverter.java} | 30 +-
.../apache/hudi/common/util/TestParquetUtils.java | 5 +-
.../hudi/metadata/TestHoodieMetadataPayload.java | 24 +-
.../hudi/metadata/TestHoodieTableMetadataUtil.java | 66 +-
.../apache/hudi/hadoop/SchemaEvolutionContext.java | 6 +
.../hudi/hadoop/avro/HoodieAvroParquetReader.java | 5 +-
.../org/apache/hudi/ColumnStatsIndexSupport.scala | 32 +-
.../org/apache/hudi/ExpressionIndexSupport.scala | 22 +-
.../apache/spark/sql/avro/SchemaConverters.scala | 5 +-
.../HoodieFileGroupReaderBasedFileFormat.scala | 10 +-
.../parquet/SparkParquetReaderBase.scala | 2 +-
.../hudi/testutils/LogFileColStatsTestUtil.java | 2 +-
.../org/apache/spark/sql/hudi/SparkHelpers.scala | 4 +-
.../org/apache/hudi/ColumnStatsIndexHelper.java | 8 +-
.../java/org/apache/hudi/TestDataSourceUtils.java | 5 +-
.../TestColStatsRecordWithMetadataRecord.java | 40 +-
.../hudi/functional/TestHoodieBackedMetadata.java | 5 +-
.../functional/TestSparkSortAndSizeClustering.java | 5 +-
.../test/resources/schema_with_logical_types.avsc | 5 +-
.../hudi/functional/ColumnStatIndexTestBase.scala | 142 ++++-
.../hudi/functional/TestColumnStatsIndex.scala | 19 +-
.../TestMetadataTableWithSparkDataSource.scala | 8 +-
.../functional/TestPartitionStatsPruning.scala | 13 +-
.../hudi/feature/index/TestExpressionIndex.scala | 16 +-
.../apache/spark/sql/adapter/Spark3_3Adapter.scala | 4 +-
.../utilities/HoodieMetadataTableValidator.java | 17 +-
.../helpers/MercifulJsonToRowConverter.java | 95 +++
.../TestHoodieMetadataTableValidator.java | 27 +-
.../deltastreamer/HoodieDeltaStreamerTestBase.java | 2 +-
.../deltastreamer/TestHoodieDeltaStreamer.java | 256 ++++++++
.../helpers/TestMercifulJsonToRowConverter.java | 65 +-
.../testutils/ColStatsUpgradeTesting.java | 160 +++++
.../testutils/sources/AbstractBaseTestSource.java | 14 +-
.../src/test/resources/col-stats/README.md | 40 ++
.../col-stats/colstats-upgrade-test-v6.zip | Bin 0 -> 212182 bytes
.../col-stats/colstats-upgrade-test-v8.zip | Bin 0 -> 260900 bytes
.../{source.avsc => source-timestamp-millis.avsc} | 4 +-
packaging/bundle-validation/validate.sh | 4 +-
scripts/release/validate_source_copyright.sh | 2 +-
100 files changed, 4376 insertions(+), 667 deletions(-)
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/stats/SparkValueMetadataUtils.java
copy
packaging/hudi-datahub-sync-bundle/src/main/java/org/apache/hudi/datahub/bundle/Main.java
=> hudi-common/src/main/java/org/apache/hudi/ParquetAdapter.java (52%)
rename hudi-common/src/main/java/org/apache/hudi/{common/model =>
stats}/HoodieColumnRangeMetadata.java (69%)
create mode 100644
hudi-common/src/main/java/org/apache/hudi/stats/ValueMetadata.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/stats/ValueType.java
create mode 100644
hudi-common/src/main/java/org/apache/parquet/schema/LogicalTypeParquetAdapter.java
create mode 100644
hudi-common/src/main/java/org/apache/parquet/schema/OriginalTypeParquetAdapter.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/stats/TestValueMetadata.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/stats/TestValueType.java
create mode 100644
hudi-hadoop-common/src/main/java/org/apache/parquet/avro/AvroSchemaConverterWithTimestampNTZ.java
create mode 100644
hudi-hadoop-common/src/main/java/org/apache/parquet/avro/HoodieAvroParquetSchemaConverter.java
copy
hudi-hadoop-common/src/{test/java/org/apache/hudi/storage/hadoop/TestHadoopStorageConfiguration.java
=> main/java/org/apache/parquet/avro/NativeAvroSchemaConverter.java} (58%)
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/ColStatsUpgradeTesting.java
create mode 100644 hudi-utilities/src/test/resources/col-stats/README.md
create mode 100644
hudi-utilities/src/test/resources/col-stats/colstats-upgrade-test-v6.zip
create mode 100644
hudi-utilities/src/test/resources/col-stats/colstats-upgrade-test-v8.zip
copy hudi-utilities/src/test/resources/streamer-config/{source.avsc =>
source-timestamp-millis.avsc} (98%)
