This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch dependabot/gradle/kafka-3.8.0
in repository https://gitbox.apache.org/repos/asf/iceberg.git
discard ee7badcb63 Build: Bump kafka from 3.7.1 to 3.8.0
add 265bce76f7 Build: Bump software.amazon.awssdk:bom from 2.26.21 to
2.26.25 (#10800)
add 31d52c0edb Build: Bump nessie from 0.93.1 to 0.94.2 (#10798)
add 349046889a Build: Bump net.snowflake:snowflake-jdbc from 3.17.0 to
3.18.0 (#10801)
add b8b57b2ff7 Build: Bump mkdocs-material from 9.5.29 to 9.5.30 (#10796)
add 2c42e67375 Flink: Disabling flaky test
TestIcebergSourceFailover.testBoundedWithSavepoint (#10802)
add 94d336ce01 Build: Bump mkdocs-awesome-pages-plugin from 2.9.2 to 2.9.3
(#10795)
add 30e761ebe4 Kafka Connect: Runtime distribution with integration tests
(#10739)
add e0a464f1cf Flink: improve snapshot compatibility check by comparing
projected sort schema in SortKeySerializer. also add unit tests for serializer
snapshot. (#10794)
add f7585932a6 Flink: support limit pushdown in FLIP-27 source (#10748)
add 96587abf7f Flink: Remove MiniClusterResource (#10817)
add 4d1ceac275 Docs: Use link addresses instead of descriptions in
releases.md (#10815)
add 0ff90e7732 Build: Declare avro as an api dependency of iceberg-core
(#10573)
add 72b39ab91d Flink: backport PR #10748 for limit pushdown (#10813)
add 76dba8fe83 Docs: Fix header for entries metadata table (#10826)
add 506fee4926 Spark 3.5: Support Reporting Column Stats (#10659)
add 84c9125173 Flink: Backport #10548 to v1.18 and v1.17 (#10776)
add 806da5cfc7 Infra: Improve feature request template (#10825)
add 99b8e88a88 Core: Replace the duplicated ALL_DATA_FILES with
ALL_DELETE_FILES (#10836)
add eb9d3951ee Core: Adds Basic Classes for Iceberg Table Version 3
(#10760)
add 39373d09c2 Core: Allow SnapshotProducer to skip uncommitted manifest
cleanup after commit (#10523)
add 6e7113a529 Flink: a few small fixes or tuning for range partitioner
(#10823)
add 9a67f0b85c Drop support for Java 8 (#10518)
add 57a9698d7b Build: Bump kafka from 3.7.1 to 3.8.0
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (ee7badcb63)
\
N -- N -- N refs/heads/dependabot/gradle/kafka-3.8.0 (57a9698d7b)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/ISSUE_TEMPLATE/iceberg_improvement.yml | 10 +-
.github/workflows/delta-conversion-ci.yml | 4 +-
.github/workflows/flink-ci.yml | 2 +-
.github/workflows/hive-ci.yml | 4 +-
.github/workflows/java-ci.yml | 6 +-
.github/workflows/publish-snapshot.yml | 2 +-
.github/workflows/spark-ci.yml | 2 +-
README.md | 2 +-
build.gradle | 54 ++---
.../apache/iceberg/BaseUpdatePartitionSpec.java | 2 +-
.../main/java/org/apache/iceberg/FastAppend.java | 10 +
.../java/org/apache/iceberg/ManifestFiles.java | 4 +
.../org/apache/iceberg/ManifestListWriter.java | 35 ++++
.../java/org/apache/iceberg/ManifestLists.java | 3 +
.../java/org/apache/iceberg/ManifestWriter.java | 73 +++++++
.../java/org/apache/iceberg/SnapshotProducer.java | 43 ++--
.../java/org/apache/iceberg/TableMetadata.java | 2 +-
.../iceberg/{V2Metadata.java => V3Metadata.java} | 12 +-
.../apache/iceberg/DeleteFileIndexTestBase.java | 2 +-
.../apache/iceberg/MetadataTableScanTestBase.java | 2 +-
.../iceberg/ScanPlanningAndReportingTestBase.java | 2 +-
.../src/test/java/org/apache/iceberg/TestBase.java | 2 +-
.../java/org/apache/iceberg/TestBatchScans.java | 2 +-
.../org/apache/iceberg/TestCommitReporting.java | 2 +-
.../org/apache/iceberg/TestCreateTransaction.java | 2 +-
.../apache/iceberg/TestEntriesMetadataTable.java | 2 +-
.../java/org/apache/iceberg/TestFastAppend.java | 52 ++++-
.../java/org/apache/iceberg/TestFindFiles.java | 2 +-
.../org/apache/iceberg/TestFormatVersions.java | 2 +-
.../iceberg/TestIncrementalDataTableScan.java | 2 +-
.../org/apache/iceberg/TestLocalFilterFiles.java | 2 +-
.../org/apache/iceberg/TestLocationProvider.java | 2 +-
.../org/apache/iceberg/TestManifestCleanup.java | 2 +-
.../apache/iceberg/TestManifestReaderStats.java | 2 +-
.../org/apache/iceberg/TestManifestWriter.java | 2 +-
.../apache/iceberg/TestMetadataTableFilters.java | 4 +-
.../org/apache/iceberg/TestMetadataTableScans.java | 46 ++---
...stMetadataTableScansWithPartitionEvolution.java | 3 +-
.../test/java/org/apache/iceberg/TestMetrics.java | 2 +-
.../java/org/apache/iceberg/TestMetricsModes.java | 2 +-
.../org/apache/iceberg/TestMicroBatchBuilder.java | 2 +-
.../org/apache/iceberg/TestPartitionSpecInfo.java | 2 +-
.../org/apache/iceberg/TestReplaceTransaction.java | 2 +-
.../org/apache/iceberg/TestRewriteManifests.java | 2 +-
.../java/org/apache/iceberg/TestScanSummary.java | 2 +-
.../iceberg/TestScansAndSchemaEvolution.java | 2 +-
.../apache/iceberg/TestSchemaAndMappingUpdate.java | 2 +-
.../test/java/org/apache/iceberg/TestSchemaID.java | 2 +-
.../apache/iceberg/TestSetPartitionStatistics.java | 2 +-
.../java/org/apache/iceberg/TestSetStatistics.java | 2 +-
.../test/java/org/apache/iceberg/TestSnapshot.java | 2 +-
.../org/apache/iceberg/TestSnapshotLoading.java | 2 +-
.../org/apache/iceberg/TestSnapshotManager.java | 2 +-
.../org/apache/iceberg/TestSnapshotSelection.java | 2 +-
.../org/apache/iceberg/TestSnapshotSummary.java | 2 +-
.../java/org/apache/iceberg/TestSortOrder.java | 2 +-
.../java/org/apache/iceberg/TestSplitPlanning.java | 2 +-
.../iceberg/TestTableMetadataSerialization.java | 2 +-
.../iceberg/TestTableUpdatePartitionSpec.java | 2 +-
.../apache/iceberg/TestTimestampPartitions.java | 2 +-
.../java/org/apache/iceberg/TestTransaction.java | 2 +-
.../apache/iceberg/TestUpdatePartitionSpec.java | 2 +-
.../java/org/apache/iceberg/TestWapWorkflow.java | 2 +-
.../iceberg/actions/TestSizeBasedRewriter.java | 2 +-
.../apache/iceberg/io/TestOutputFileFactory.java | 2 +-
.../apache/iceberg/mapping/TestMappingUpdates.java | 2 +-
.../resources/TableMetadataUnsupportedVersion.json | 4 +-
deploy.gradle | 4 +-
docs/docs/spark-queries.md | 1 +
.../flink/sink/shuffle/SortKeySerializer.java | 23 ++-
.../apache/iceberg/flink/source/IcebergSource.java | 3 +-
.../enumerator/AbstractIcebergEnumerator.java | 11 +-
...derFunction.java => LimitableDataIterator.java} | 41 ++--
.../iceberg/flink/source/reader/RecordLimiter.java | 30 +--
.../flink/source/reader/RowDataReaderFunction.java | 40 +++-
.../apache/iceberg/flink/MiniClusterResource.java | 53 -----
.../java/org/apache/iceberg/flink/TestBase.java | 2 +-
.../apache/iceberg/flink/TestFlinkTableSink.java | 2 +-
.../org/apache/iceberg/flink/TestFlinkUpsert.java | 2 +-
.../apache/iceberg/flink/TestIcebergConnector.java | 2 +-
.../TestBucketPartitionerFlinkIcebergSink.java | 4 +-
.../iceberg/flink/sink/TestFlinkIcebergSink.java | 7 +-
.../flink/sink/TestFlinkIcebergSinkBranch.java | 4 +-
.../iceberg/flink/sink/TestFlinkIcebergSinkV2.java | 5 +-
.../flink/sink/TestFlinkIcebergSinkV2Branch.java | 4 +-
.../shuffle/TestSortKeySerializerSnapshot.java | 213 +++++++++++++++++++
.../flink/source/ChangeLogTableTestBase.java | 4 +-
.../apache/iceberg/flink/source/TestFlinkScan.java | 2 +-
.../flink/source/TestFlinkSourceConfig.java | 7 +-
.../flink/source/TestIcebergSourceFailover.java | 2 +
.../TestIcebergSourceWithWatermarkExtractor.java | 2 +-
.../TestIcebergSpeculativeExecutionSupport.java | 10 +-
.../apache/iceberg/flink/source/TestSqlBase.java | 2 +-
.../iceberg/flink/source/TestStreamScanSql.java | 4 +-
.../source/reader/TestLimitableDataIterator.java | 84 ++++++++
.../flink/sink/shuffle/SortKeySerializer.java | 23 ++-
.../apache/iceberg/flink/source/IcebergSource.java | 3 +-
.../enumerator/AbstractIcebergEnumerator.java | 11 +-
.../source/reader/LimitableDataIterator.java} | 41 ++--
.../iceberg/flink/source/reader/RecordLimiter.java | 30 +--
.../flink/source/reader/RowDataReaderFunction.java | 40 +++-
.../apache/iceberg/flink/MiniClusterResource.java | 68 -------
.../java/org/apache/iceberg/flink/TestBase.java | 2 +-
.../apache/iceberg/flink/TestFlinkTableSink.java | 2 +-
.../org/apache/iceberg/flink/TestFlinkUpsert.java | 2 +-
.../apache/iceberg/flink/TestIcebergConnector.java | 2 +-
.../TestBucketPartitionerFlinkIcebergSink.java | 4 +-
.../iceberg/flink/sink/TestFlinkIcebergSink.java | 7 +-
.../flink/sink/TestFlinkIcebergSinkBranch.java | 4 +-
.../iceberg/flink/sink/TestFlinkIcebergSinkV2.java | 5 +-
.../flink/sink/TestFlinkIcebergSinkV2Branch.java | 4 +-
.../shuffle/TestSortKeySerializerSnapshot.java | 213 +++++++++++++++++++
.../flink/source/ChangeLogTableTestBase.java | 4 +-
.../apache/iceberg/flink/source/TestFlinkScan.java | 2 +-
.../flink/source/TestFlinkSourceConfig.java | 7 +-
.../flink/source/TestIcebergSourceContinuous.java | 2 +-
.../flink/source/TestIcebergSourceFailover.java | 2 +
.../TestIcebergSourceWithWatermarkExtractor.java | 2 +-
.../TestIcebergSpeculativeExecutionSupport.java | 10 +-
.../apache/iceberg/flink/source/TestSqlBase.java | 2 +-
.../iceberg/flink/source/TestStreamScanSql.java | 4 +-
.../source/reader/TestLimitableDataIterator.java | 84 ++++++++
.../sink/shuffle/AggregatedStatisticsTracker.java | 23 ++-
.../flink/sink/shuffle/CompletedStatistics.java | 8 +
.../sink/shuffle/DataStatisticsCoordinator.java | 45 +++--
.../flink/sink/shuffle/RangePartitioner.java | 8 +-
.../flink/sink/shuffle/SketchRangePartitioner.java | 19 +-
.../iceberg/flink/sink/shuffle/SketchUtil.java | 17 ++
.../flink/sink/shuffle/SortKeySerializer.java | 31 +--
.../iceberg/flink/sink/shuffle/SortKeyUtil.java | 59 ++++++
.../apache/iceberg/flink/source/IcebergSource.java | 3 +-
.../source/reader/LimitableDataIterator.java} | 41 ++--
.../iceberg/flink/source/reader/RecordLimiter.java | 30 +--
.../flink/source/reader/RowDataReaderFunction.java | 40 +++-
.../apache/iceberg/flink/MiniClusterResource.java | 68 -------
.../java/org/apache/iceberg/flink/TestBase.java | 2 +-
.../apache/iceberg/flink/TestFlinkTableSink.java | 2 +-
.../org/apache/iceberg/flink/TestFlinkUpsert.java | 2 +-
.../apache/iceberg/flink/TestIcebergConnector.java | 2 +-
.../maintenance/operator/OperatorTestBase.java | 4 +-
.../TestBucketPartitionerFlinkIcebergSink.java | 4 +-
.../iceberg/flink/sink/TestFlinkIcebergSink.java | 7 +-
.../flink/sink/TestFlinkIcebergSinkBranch.java | 4 +-
.../iceberg/flink/sink/TestFlinkIcebergSinkV2.java | 5 +-
.../flink/sink/TestFlinkIcebergSinkV2Branch.java | 4 +-
.../flink/sink/shuffle/TestRangePartitioner.java | 65 ++++++
.../sink/shuffle/TestSketchRangePartitioner.java | 88 ++++++++
.../iceberg/flink/sink/shuffle/TestSketchUtil.java | 64 +++++-
.../shuffle/TestSortKeySerializerSnapshot.java | 213 +++++++++++++++++++
.../flink/sink/shuffle/TestSortKeyUtil.java | 73 +++++++
.../flink/source/ChangeLogTableTestBase.java | 4 +-
.../apache/iceberg/flink/source/TestFlinkScan.java | 2 +-
.../flink/source/TestFlinkSourceConfig.java | 7 +-
.../flink/source/TestIcebergSourceFailover.java | 2 +
.../TestIcebergSourceWithWatermarkExtractor.java | 2 +-
.../TestIcebergSpeculativeExecutionSupport.java | 4 +-
.../apache/iceberg/flink/source/TestSqlBase.java | 2 +-
.../iceberg/flink/source/TestStreamScanSql.java | 4 +-
.../source/reader/TestLimitableDataIterator.java | 84 ++++++++
gradle/libs.versions.toml | 11 +-
hive-runtime/build.gradle | 2 +-
jmh.gradle | 4 +-
kafka-connect/build.gradle | 184 ++++++++++++++++-
.../docker/docker-compose.yml | 107 ++++++++++
.../connect/IntegrationDynamicTableTest.java | 135 +++++++++++++
.../iceberg/connect/IntegrationMultiTableTest.java | 140 +++++++++++++
.../apache/iceberg/connect/IntegrationTest.java | 225 +++++++++++++++++++++
.../iceberg/connect/IntegrationTestBase.java | 155 ++++++++++++++
.../apache/iceberg/connect/KafkaConnectUtils.java | 117 +++++++++++
.../org/apache/iceberg/connect/TestContext.java | 116 +++++++++++
.../java/org/apache/iceberg/connect/TestEvent.java | 113 +++++++++++
.../src/main/resources/iceberg.png | Bin
.../src/main/resources/manifest.json | 47 +++++
.../apache/iceberg/connect/IcebergSinkConfig.java | 2 +-
.../apache/iceberg/connect/IcebergSinkTask.java | 10 +-
settings.gradle | 4 +
site/docs/contribute.md | 2 +-
site/docs/releases.md | 4 +-
site/requirements.txt | 4 +-
.../org/apache/iceberg/spark/SparkReadConf.java | 8 +
.../apache/iceberg/spark/SparkSQLProperties.java | 4 +
.../iceberg/spark/source/SparkChangelogScan.java | 2 +-
.../spark/source/SparkColumnStatistics.java | 88 ++++++++
.../org/apache/iceberg/spark/source/SparkScan.java | 54 ++++-
.../org/apache/iceberg/spark/source/Stats.java | 12 +-
.../apache/iceberg/spark/source/TestSparkScan.java | 183 +++++++++++++++++
tasks.gradle | 8 +-
187 files changed, 3765 insertions(+), 633 deletions(-)
copy core/src/main/java/org/apache/iceberg/{V2Metadata.java =>
V3Metadata.java} (98%)
copy
flink/v1.17/flink/src/main/java/org/apache/iceberg/flink/source/reader/{DataIteratorReaderFunction.java
=> LimitableDataIterator.java} (50%)
copy core/src/test/java/org/apache/iceberg/util/FakeTicker.java =>
flink/v1.17/flink/src/main/java/org/apache/iceberg/flink/source/reader/RecordLimiter.java
(62%)
delete mode 100644
flink/v1.17/flink/src/test/java/org/apache/iceberg/flink/MiniClusterResource.java
create mode 100644
flink/v1.17/flink/src/test/java/org/apache/iceberg/flink/sink/shuffle/TestSortKeySerializerSnapshot.java
copy flink/{v1.19 =>
v1.17}/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSpeculativeExecutionSupport.java
(95%)
create mode 100644
flink/v1.17/flink/src/test/java/org/apache/iceberg/flink/source/reader/TestLimitableDataIterator.java
copy
flink/{v1.17/flink/src/main/java/org/apache/iceberg/flink/source/reader/DataIteratorReaderFunction.java
=>
v1.18/flink/src/main/java/org/apache/iceberg/flink/source/reader/LimitableDataIterator.java}
(50%)
copy core/src/test/java/org/apache/iceberg/util/FakeTicker.java =>
flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/source/reader/RecordLimiter.java
(62%)
delete mode 100644
flink/v1.18/flink/src/test/java/org/apache/iceberg/flink/MiniClusterResource.java
create mode 100644
flink/v1.18/flink/src/test/java/org/apache/iceberg/flink/sink/shuffle/TestSortKeySerializerSnapshot.java
copy flink/{v1.19 =>
v1.18}/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSpeculativeExecutionSupport.java
(95%)
create mode 100644
flink/v1.18/flink/src/test/java/org/apache/iceberg/flink/source/reader/TestLimitableDataIterator.java
create mode 100644
flink/v1.19/flink/src/main/java/org/apache/iceberg/flink/sink/shuffle/SortKeyUtil.java
copy
flink/{v1.17/flink/src/main/java/org/apache/iceberg/flink/source/reader/DataIteratorReaderFunction.java
=>
v1.19/flink/src/main/java/org/apache/iceberg/flink/source/reader/LimitableDataIterator.java}
(50%)
copy core/src/test/java/org/apache/iceberg/util/FakeTicker.java =>
flink/v1.19/flink/src/main/java/org/apache/iceberg/flink/source/reader/RecordLimiter.java
(62%)
delete mode 100644
flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/MiniClusterResource.java
create mode 100644
flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/sink/shuffle/TestRangePartitioner.java
create mode 100644
flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/sink/shuffle/TestSketchRangePartitioner.java
create mode 100644
flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/sink/shuffle/TestSortKeySerializerSnapshot.java
create mode 100644
flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/sink/shuffle/TestSortKeyUtil.java
create mode 100644
flink/v1.19/flink/src/test/java/org/apache/iceberg/flink/source/reader/TestLimitableDataIterator.java
create mode 100644
kafka-connect/kafka-connect-runtime/docker/docker-compose.yml
create mode 100644
kafka-connect/kafka-connect-runtime/src/integration/java/org/apache/iceberg/connect/IntegrationDynamicTableTest.java
create mode 100644
kafka-connect/kafka-connect-runtime/src/integration/java/org/apache/iceberg/connect/IntegrationMultiTableTest.java
create mode 100644
kafka-connect/kafka-connect-runtime/src/integration/java/org/apache/iceberg/connect/IntegrationTest.java
create mode 100644
kafka-connect/kafka-connect-runtime/src/integration/java/org/apache/iceberg/connect/IntegrationTestBase.java
create mode 100644
kafka-connect/kafka-connect-runtime/src/integration/java/org/apache/iceberg/connect/KafkaConnectUtils.java
create mode 100644
kafka-connect/kafka-connect-runtime/src/integration/java/org/apache/iceberg/connect/TestContext.java
create mode 100644
kafka-connect/kafka-connect-runtime/src/integration/java/org/apache/iceberg/connect/TestEvent.java
copy site/docs/assets/images/iceberg-logo-icon.png =>
kafka-connect/kafka-connect-runtime/src/main/resources/iceberg.png (100%)
create mode 100644
kafka-connect/kafka-connect-runtime/src/main/resources/manifest.json
create mode 100644
spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkColumnStatistics.java
