This is an automated email from the ASF dual-hosted git repository.
github-actions[bot] pushed a change to branch bot/trino-pin
in repository https://gitbox.apache.org/repos/asf/hudi.git
discard b932ede57fef chore(trino): advance trino master pin to 92757c4365a1
add d43fa7fb6319 fix(debezium): backfill Postgres TOAST columns on a copy
so the merge result survives (#19749)
add 0c3f00ee0958 fix(trino): read LSM archived-timeline parquet files
through a Trino-… (#19732)
add 98c61edfc674 test(utilities): cover the KafkaAvroSchemaDeserializer
schema-refresh... (#19760)
add 3eef646d3afd fix(hive): read the skeleton file when a bootstrap query
projects no columns (#19510)
add 85033e339f66 test(spark): add write and commit path coverage (#19163)
add 87778deec6f6 fix(streamer): Include start commit in S3/GCS IncrSource
incremental query (#18949)
add fe82430e694a fix(sync): keep each partition's recorded location when
cascading Glue schema changes (#19761)
add 1895de8a0648 perf(flink): preempt inactive write buckets on memory
exhaustion (#19728)
add d4f9eb930a47 refactor(variant): self-align log-block variant rows,
drop buffer-level projection hook (#18923)
add 863f84a08864 fix(sync): drop the duplicate UpdateTable in
updateTableSchema (#19762)
add a0d4670ad829 fix: close JDBC connection in UtilHelpers.getJDBCSchema
to prevent connection leak (#19772)
add 2d73fe40dd8a feat(spark): support bucket index for LSM tables (#19727)
add 452b237501df fix(lock): name the cause on FAILED_TO_RELEASE in
StorageBasedLockProvider (#19574)
add 3a3a32fb48f4 docs(hudi-notebooks): note S3A works with any
S3-compatible object store (#19423)
add efe02e1621c1 fix(variant): close the shredded-read gaps exposed by a
mixed-layout test matrix (#19687)
new 649f1ef6dec1 chore(trino): advance trino master pin to b46814cbf405
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (b932ede57fef)
\
N -- N -- N refs/heads/bot/trino-pin (649f1ef6dec1)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hudi/aws/sync/AWSGlueCatalogSyncClient.java | 68 +-
.../hudi/aws/sync/TestAWSGlueSyncClient.java | 103 +-
.../transaction/lock/StorageBasedLockProvider.java | 56 +-
.../table/BucketSortBulkInsertPartitioner.java | 12 +-
.../lock/TestStorageBasedLockProvider.java | 41 +-
.../run/strategy/JavaExecutionStrategy.java | 6 +-
.../JavaCustomColumnsSortPartitioner.java | 1 +
.../MultipleSparkJobExecutionStrategy.java | 16 +-
.../client/utils/SparkInternalSchemaConverter.java | 33 +-
.../BucketIndexBulkInsertPartitionerWithRows.java | 37 +-
...ntBucketIndexBulkInsertPartitionerWithRows.java | 4 +
.../bulkinsert/RDDBucketIndexPartitioner.java | 5 +-
.../RDDConsistentBucketBulkInsertPartitioner.java | 4 +
.../RDDCustomColumnsSortPartitioner.java | 2 +
.../RowCustomColumnsSortPartitioner.java | 4 +
.../SpatialCurveSortPartitionerBase.java | 49 +
.../hudi/io/storage/HoodieSparkParquetReader.java | 21 +-
.../SparkFileFormatInternalRowReaderContext.scala | 82 +-
.../apache/spark/sql/BucketPartitionUtils.scala | 42 +-
.../TestRowCustomColumnsSortPartitioner.java | 28 +
.../TestSpatialCurveSortPartitionerBase.java | 78 ++
.../hudi/common/config/HoodieStorageConfig.java | 6 +-
.../hudi/common/engine/HoodieReaderContext.java | 17 +-
.../debezium/PostgresDebeziumAvroPayload.java | 33 +-
.../apache/hudi/common/schema/HoodieSchema.java | 18 +-
.../table/log/block/HoodieAvroDataBlock.java | 5 +-
.../table/read/buffer/FileGroupRecordBuffer.java | 26 +-
.../buffer/PositionBasedFileGroupRecordBuffer.java | 10 +-
.../org/apache/hudi/common/util/SortUtils.java | 153 +++
.../core/io/storage/HoodieBootstrapFileReader.java | 8 +-
.../hudi/common/avro/TestVariantSchemaUtils.java | 4 +-
.../hudi/common/schema/TestHoodieSchema.java | 8 +-
.../org/apache/hudi/common/util/TestSortUtils.java | 112 ++
.../resources/variant_backward_compat/README.md | 14 +
.../variant_shredded_mixed_cow.zip | Bin 0 -> 88703 bytes
.../org/apache/hudi/sink/StreamWriteFunction.java | 73 +-
.../sink/buffer/PreemptiveMemorySegmentPool.java | 135 +++
.../hudi/sink/bulk/sort/SortOperatorGen.java | 43 +
.../hudi/sink/clustering/ClusteringOperator.java | 21 +-
.../TestBucketStreamWriteMemoryExhaustion.java | 178 ++-
.../buffer/TestPreemptiveMemorySegmentPool.java | 156 +++
.../hudi/sink/bulk/sort/TestSortOperatorGen.java | 29 +
.../sink/clustering/TestClusteringOperator.java | 63 +-
.../ITTestVariantCrossEngineCompatibility.java | 47 +
.../parquet/avro/TestAvroSchemaConverter.java | 4 +-
.../hudi/hadoop/HiveHoodieReaderContext.java | 144 +++
.../hudi/hadoop/HoodieColumnProjectionUtils.java | 88 +-
.../hudi/hadoop/HoodieParquetInputFormat.java | 337 +++++-
.../hudi/hadoop/TestHiveHoodieReaderContext.java | 235 ++++
.../hadoop/TestHoodieColumnProjectionUtils.java | 27 +
.../hudi/hadoop/TestHoodieParquetInputFormat.java | 395 +++++++
.../realtime/TestHoodieRealtimeRecordReader.java | 56 +
.../hudi/hadoop/testutils/InputFormatTestUtil.java | 247 ++++
hudi-notebooks/README.md | 36 +
.../BaseDatasetBulkInsertCommitActionExecutor.java | 4 +-
.../DatasetBucketRescaleCommitActionExecutor.java | 4 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 9 +-
.../HoodieFileGroupReaderBasedFileFormat.scala | 52 +-
.../parquet/ParquetSchemaEvolutionUtils.scala | 155 ++-
.../parquet/TestParquetSchemaEvolutionUtils.scala | 209 ++++
.../parquet/VariantParquetTestFixtures.scala | 77 ++
.../procedures/RunClusteringProcedure.scala | 75 +-
.../bulkinsert/TestLSMBulkInsertPartitioner.java | 40 +
.../functional/TestHiveTableSchemaEvolution.java | 4 +-
.../TestSparkConsistentBucketClustering.java | 5 +-
.../TestIncrementalReadWithFileGroupReader.scala | 189 +++
.../apache/hudi/functional/TestLSMDataSource.scala | 120 +-
.../functional/TestLegacyParquetReadPath.scala | 65 +-
.../TestPostgresDebeziumToastV6ReadMerge.scala | 124 ++
.../sql/hudi/common/HoodieSparkSqlTestBase.scala | 32 +-
.../sql/hudi/dml/insert/TestInsertTable2.scala | 162 ++-
.../sql/hudi/dml/schema/TestVariantDataType.scala | 540 ++++-----
.../schema/TestVariantShreddingMixedLayouts.scala | 1211 ++++++++++++++++++++
.../dml/schema/VariantShreddingTestSupport.scala | 717 ++++++++++++
.../hudi/procedure/TestClusteringProcedure.scala | 180 ++-
.../Spark3LegacyHoodieParquetFileFormat.scala | 6 +
.../TestHoodieVariantReconstructionRoundTrip.java | 166 +++
.../parquet/Spark40HoodieParquetReadSupport.scala | 91 +-
.../Spark40LegacyHoodieParquetFileFormat.scala | 6 +
.../datasources/parquet/Spark40ParquetReader.scala | 8 +
.../TestSpark40HoodieParquetReadSupport.scala | 142 +++
.../Spark41LegacyHoodieParquetFileFormat.scala | 6 +
.../Spark42LegacyHoodieParquetFileFormat.scala | 6 +
.../plugin/hudi/io/HudiTrinoFileReaderFactory.java | 2 +-
.../plugin/hudi/io/TrinoParquetFileReader.java | 443 +++++++
.../io/trino/plugin/hudi/TestHudiSmokeTest.java | 25 +
.../plugin/hudi/io/TestTrinoParquetFileReader.java | 252 ++++
.../testing/ResourceHudiTablesInitializer.java | 1 +
.../src/test/resources/archived_timeline.parquet | Bin 0 -> 3263 bytes
.../hudi_cow_archived_timeline.md | 124 ++
.../hudi_mor_archived_timeline.md | 96 ++
.../hudi_mor_archived_timeline.zip | Bin 0 -> 242504 bytes
.../org/apache/hudi/utilities/UtilHelpers.java | 35 +-
.../utilities/sources/helpers/QueryRunner.java | 13 +-
.../sources/S3EventsHoodieIncrSourceHarness.java | 20 +-
.../utilities/sources/TestAvroKafkaSource.java | 91 ++
.../sources/TestGcsEventsHoodieIncrSource.java | 118 +-
.../sources/TestS3EventsHoodieIncrSource.java | 106 +-
pom.xml | 2 +-
99 files changed, 8478 insertions(+), 675 deletions(-)
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestSpatialCurveSortPartitionerBase.java
create mode 100644
hudi-common/src/test/resources/variant_backward_compat/variant_shredded_mixed_cow.zip
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/buffer/PreemptiveMemorySegmentPool.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/buffer/TestPreemptiveMemorySegmentPool.java
create mode 100644
hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/TestParquetSchemaEvolutionUtils.scala
create mode 100644
hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/VariantParquetTestFixtures.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestIncrementalReadWithFileGroupReader.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPostgresDebeziumToastV6ReadMerge.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/schema/TestVariantShreddingMixedLayouts.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/schema/VariantShreddingTestSupport.scala
create mode 100644
hudi-trino/src/main/java/io/trino/plugin/hudi/io/TrinoParquetFileReader.java
create mode 100644
hudi-trino/src/test/java/io/trino/plugin/hudi/io/TestTrinoParquetFileReader.java
create mode 100644 hudi-trino/src/test/resources/archived_timeline.parquet
create mode 100644
hudi-trino/src/test/resources/hudi-testing-data/hudi_cow_archived_timeline.md
create mode 100644
hudi-trino/src/test/resources/hudi-testing-data/hudi_mor_archived_timeline.md
create mode 100644
hudi-trino/src/test/resources/hudi-testing-data/hudi_mor_archived_timeline.zip
