This is an automated email from the ASF dual-hosted git repository.
vinishjail97 pushed a change to branch hudi-upgrade-version
in repository https://gitbox.apache.org/repos/asf/incubator-xtable.git
from 8d3c7e01 Address self review comments and link GH issues for failing
tests
add 22f4026f Delta source: extract nested field's stats (#769)
add 81af6e29 Fix Delta to Iceberg not working on column mapping enabled
Delta source table (#766)
add f9ee9489 Implement Paimon Source Incremental Sync (#780)
add 50ae6116 [787] Add maven wrapper for consistent builds (#788)
add 7a1c5b1a [755] Support column stats for paimon (#767)
add b8fce01b fix scope of spark catalyst in utilities bundle (#790)
add 559a9518 Upgrade Iceberg to 1.9.2 and Avro to 1.12.0 (#784)
add 8cab6a29 fix(delta): avoid NPE for binary in map/array schemas (#795)
add 8e583674 Remove redundant getSnapshotAt calls per commit (#791)
add abbf4b72 fix(iceberg): nested comments with qualified name (#797)
add fe7215e2 add .sdkmanrc to .gitignore (#793)
add 5c25674b Remove wildcard imports and enforce with spotless (#809)
add 43075655 Parquet source: Expand column stats support, fix bugs in
schema conversion (#805)
add f991e315 [bug] Parquet Source: snapshot sync fails on multiple commits
with partitions (#806)
add c0205955 Enable Delta checkpoint CRC exception throwing by default
(#817)
add 62453c7e Bump actions/setup-node from 5 to 6 (#753)
add 2ae96426 Handle empty EarliestCommitToRetain (#732)
add 634bcb6b [bug] Fix batch INSERT_OVERWRITE replacecommits dropping adds
in HudiDataFileExtractor (#816)
add f2ba892f Fallback to parquet footers when metadata table col stats are
missing (#818)
add fec7f906 Merge branch 'main' into hudi-upgrade-version
add 4058fb8f Upgrade hudi to 1.2.0 and align spark/delta with main
No new revisions were added by this update.
Summary of changes:
.github/workflows/mvn-ci-build.yml | 2 +-
.github/workflows/mvn-license-check.yml | 2 +-
.github/workflows/package-deploy.yml | 4 +-
.github/workflows/site-build.yml | 2 +-
.github/workflows/site-deploy.yml | 2 +-
.gitignore | 5 +-
.../wrapper/maven-wrapper.properties | 8 +-
Dockerfile | 8 +-
README.md | 14 +-
demo/start_demo.sh | 2 +-
mvnw | 295 +++++++
mvnw.cmd | 189 +++++
pom.xml | 28 +-
release/release_guide.md | 8 +-
release/scripts/cut_release_branch.sh | 4 +-
release/scripts/deploy_staging_jars.sh | 2 +-
release/scripts/validate_source_rat.sh | 2 +-
website/docs/setup.md | 4 +-
.../apache/xtable/model/schema/InternalField.java | 3 +
.../org/apache/xtable/model/storage/FilesDiff.java | 6 +-
.../TestThreePartHierarchicalTableIdentifier.java | 2 +-
.../apache/xtable/spi/sync/TestCatalogSync.java | 2 +-
.../src/main/resources/META-INF/LICENSE-bundled | 6 +-
xtable-core/pom.xml | 2 +-
.../apache/xtable/delta/DeltaConversionSource.java | 10 +-
.../apache/xtable/delta/DeltaConversionUtils.java | 3 +-
.../xtable/delta/DeltaDataFileExtractor.java | 2 +-
.../delta/DeltaDataFileUpdatesExtractor.java | 10 +-
.../apache/xtable/delta/DeltaSchemaExtractor.java | 9 +-
.../apache/xtable/delta/DeltaTableExtractor.java | 23 +
.../apache/xtable/hudi/HudiConversionSource.java | 24 +
.../apache/xtable/hudi/HudiConversionTarget.java | 2 +-
.../apache/xtable/hudi/HudiDataFileExtractor.java | 4 +-
.../apache/xtable/hudi/HudiFileStatsExtractor.java | 101 ++-
.../org/apache/xtable/hudi/HudiTableExtractor.java | 2 +-
.../hudi/PathBasedPartitionValuesExtractor.java | 6 +-
.../xtable/iceberg/IcebergConversionTarget.java | 53 ++
.../xtable/iceberg/IcebergDataFileUpdatesSync.java | 15 +-
.../xtable/iceberg/IcebergSchemaExtractor.java | 59 +-
.../apache/xtable/iceberg/IcebergSchemaSync.java | 14 +-
.../apache/xtable/iceberg/IcebergTableManager.java | 12 +-
.../kernel/DeltaKernelPartitionExtractor.java | 6 +-
.../xtable/paimon/PaimonConversionSource.java | 106 ++-
.../xtable/paimon/PaimonDataFileExtractor.java | 88 ++-
.../apache/xtable/paimon/PaimonStatsExtractor.java | 187 +++++
.../xtable/parquet/ParquetConversionSource.java | 57 +-
.../xtable/parquet/ParquetMetadataExtractor.java | 24 +-
.../parquet/ParquetPartitionValueExtractor.java | 15 +-
.../xtable/parquet/ParquetSchemaExtractor.java | 65 +-
.../xtable/parquet/ParquetStatsConverterUtil.java | 69 --
.../xtable/parquet/ParquetStatsExtractor.java | 180 ++---
.../org/apache/xtable/ITConversionController.java | 45 +-
.../java/org/apache/xtable/TestIcebergTable.java | 4 +-
.../java/org/apache/xtable/TestPaimonTable.java | 64 +-
.../org/apache/xtable/TestSparkDeltaTable.java | 34 +-
.../java/org/apache/xtable/TestSparkHudiTable.java | 12 +
.../org/apache/xtable/ValidationTestHelper.java | 3 +-
.../conversion/TestConversionController.java | 2 +-
.../xtable/delta/ITDeltaConversionSource.java | 30 +-
.../xtable/delta/TestDeltaActionsConverter.java | 24 +-
.../org/apache/xtable/delta/TestDeltaHelper.java | 37 +
.../xtable/delta/TestDeltaPartitionExtractor.java | 3 +-
.../xtable/delta/TestDeltaSchemaExtractor.java | 75 ++
.../xtable/delta/TestDeltaStatsExtractor.java | 41 +-
.../apache/xtable/hudi/ITHudiConversionSource.java | 52 +-
.../apache/xtable/hudi/ITHudiConversionTarget.java | 2 +-
.../xtable/hudi/TestHudiConversionSource.java | 180 +++++
.../xtable/hudi/TestHudiConversionTarget.java | 4 +-
.../xtable/hudi/TestHudiFileStatsExtractor.java | 148 ++++
.../xtable/iceberg/ITIcebergConversionSource.java | 3 +
.../iceberg/TestIcebergConversionSource.java | 57 +-
.../xtable/iceberg/TestIcebergDataHelper.java | 6 +-
.../TestIcebergPartitionValueConverter.java | 3 +-
.../xtable/iceberg/TestIcebergSchemaExtractor.java | 149 +++-
.../xtable/iceberg/TestIcebergSchemaSync.java | 11 +-
.../org/apache/xtable/iceberg/TestIcebergSync.java | 3 +
.../xtable/iceberg/TestIcebergTableManager.java | 17 +-
.../kernel/ITDeltaKernelConversionSource.java | 31 +-
.../kernel/TestDeltaKernelPartitionExtractor.java | 17 +-
.../kernel/TestDeltaKernelSchemaExtractor.java | 14 +-
.../kernel/TestDeltaKernelStatsExtractor.java | 6 +-
.../xtable/paimon/TestPaimonConversionSource.java | 135 +++-
.../xtable/paimon/TestPaimonDataFileExtractor.java | 151 +++-
.../xtable/paimon/TestPaimonStatsExtractor.java | 539 +++++++++++++
.../xtable/parquet/ITParquetConversionSource.java | 241 +++---
.../xtable/parquet/TestParquetSchemaExtractor.java | 138 +++-
.../xtable/parquet/TestParquetStatsExtractor.java | 859 ++++++++++++---------
.../apache/xtable/testutil/ColumnStatMapUtil.java | 35 +
.../src/test/resources/schemas/basic_schema.avsc | 25 +-
.../src/main/resources/META-INF/LICENSE-bundled | 6 +-
.../extensions/AddFieldIdsClientInitCallback.java | 5 +-
.../HoodieAvroWriteSupportWithFieldIds.java | 3 +-
.../src/main/resources/META-INF/LICENSE-bundled | 8 +-
.../TestAddFieldIdsClientInitCallback.java | 9 +-
.../apache/xtable/hudi/idtracking/IdTracker.java | 8 +-
xtable-service/README.md | 4 +-
xtable-service/pom.xml | 2 +-
.../xtable/service/TestConversionResource.java | 4 +-
xtable-utilities/pom.xml | 5 +
.../main/resources/xtable-conversion-defaults.yaml | 4 +-
.../xtable/utilities/TestRunCatalogSync.java | 2 +-
.../org/apache/xtable/utilities/TestRunSync.java | 9 +-
102 files changed, 3865 insertions(+), 1152 deletions(-)
copy demo/presto/catalog/hive.properties =>
.mvn/wrapper/maven-wrapper.properties (82%)
create mode 100755 mvnw
create mode 100644 mvnw.cmd
create mode 100644
xtable-core/src/main/java/org/apache/xtable/paimon/PaimonStatsExtractor.java
delete mode 100644
xtable-core/src/main/java/org/apache/xtable/parquet/ParquetStatsConverterUtil.java
create mode 100644
xtable-core/src/test/java/org/apache/xtable/hudi/TestHudiConversionSource.java
create mode 100644
xtable-core/src/test/java/org/apache/xtable/paimon/TestPaimonStatsExtractor.java
